diff --git a/THIRD-PARTY-NOTICES.md b/THIRD-PARTY-NOTICES.md index ac840133..5f5b75a9 100644 --- a/THIRD-PARTY-NOTICES.md +++ b/THIRD-PARTY-NOTICES.md @@ -101,6 +101,12 @@ Windows headers, without a C runtime. Pieces of these projects that can end up i the x86 branch filter of the LZMA SDK (public domain, Igor Pavlov, https://www.7-zip.org/sdk.html), and the published app as its payload. Filee 1.x's installer was built with Inno Setup, which is no longer used. +## Unicode data + +`src/Filee.Engines/Typesetting/UnicodeData.Generated.cs` holds Line_Break, Bidi_Class, Script and East_Asian_Width +tables that `build/generate-unicode-tables.ps1` generates from the Unicode Character Database 16.0.0 (Unicode License +v3, Copyright © 1991-2024 Unicode, Inc., https://www.unicode.org/license.txt). + ## Fonts and icons | Component | License | Project | diff --git a/build/generate-unicode-tables.ps1 b/build/generate-unicode-tables.ps1 new file mode 100644 index 00000000..e4d81f43 --- /dev/null +++ b/build/generate-unicode-tables.ps1 @@ -0,0 +1,224 @@ +<# +.SYNOPSIS + Generates the Unicode property tables of Filee's layout engine + (src/Filee.Engines/Typesetting/UnicodeData.Generated.cs) from the Unicode Character Database. + +.DESCRIPTION + Writes Line_Break (UAX #14), Bidi_Class (UAX #9), Script (UAX #24) and East_Asian_Width (UAX #11) as range tables + that Typesetting/UnicodeData.cs searches. Every code point is covered: unlisted ones get the defaults the files + declare in their "@missing" lines. Each enum lists the property's overall default first (XX, L, N, Unknown), so its + value 0 is what code points outside the code space get. + + The version is pinned to UCD 16.0.0, the one .NET 10 uses for System.Globalization (CharUnicodeInfo, Rune, + StringInfo), so the layout engine and .NET agree on which characters exist and where grapheme clusters end. Move + both together: when the target framework changes its Unicode version, update $Version and the hashes below and run + the script again. + + The UCD files are downloaded from unicode.org into build/.cache and verified with SHA-256; they are not committed. + The generated C# file is committed, so building Filee needs neither the script nor the network. + +.PARAMETER Output + The C# file to write. Default: src/Filee.Engines/Typesetting/UnicodeData.Generated.cs. + +.EXAMPLE + pwsh build/generate-unicode-tables.ps1 +#> +[CmdletBinding()] +param( + [string]$Output = (Join-Path $PSScriptRoot '..\src\Filee.Engines\Typesetting\UnicodeData.Generated.cs') +) + +$ErrorActionPreference = 'Stop' +$ProgressPreference = 'SilentlyContinue' # Invoke-WebRequest is ~10x faster without the progress bar + +$Version = '16.0.0' +# UCD file => SHA-256. Update together with $Version. +$files = [ordered]@{ + 'LineBreak.txt' = 'E97E4259D0D20FAB150B9C7B4B28ABFAE5CD78CA97E7F4AC6ED20D685D5F4A7C' + 'extracted/DerivedBidiClass.txt' = '71ED943A49C58568D8D92E80ECC2BA2F06E62AEE9C8EBB0E6E8BD2C3ED8B180E' + 'Scripts.txt' = '9E88F0A677DF47311106340BE8EDE2ECDACD9C1C931831218D2BE6D5508E0039' + 'EastAsianWidth.txt' = '43ADC76C0686A42CB370764EB8CFE2B2A45B10B855E5572A2DB4A0EECCE15D5B' + 'PropertyValueAliases.txt' = '440FD3E5460B9BFE31DA67B6F923992E1989D31FE2ED91E091C4B8F8E2620BF9' +} + +$cache = Join-Path $PSScriptRoot ".cache\ucd-$Version" +New-Item -ItemType Directory -Force -Path $cache | Out-Null + +function Get-UcdFile([string]$name) { + $expected = $files[$name] + $file = Join-Path $cache ($name -replace '/', '_') + if ((Test-Path $file) -and (Get-FileHash $file -Algorithm SHA256).Hash -eq $expected) { + Write-Host "ok $name (cached)" + return $file + } + $url = "https://www.unicode.org/Public/$Version/ucd/$name" + Write-Host "get $url" + Invoke-WebRequest -Uri $url -OutFile $file -UseBasicParsing + $actual = (Get-FileHash $file -Algorithm SHA256).Hash + if ($actual -ne $expected) { + Remove-Item $file + throw "Checksum mismatch for $name. Expected $expected, got $actual." + } + return $file +} + +# ───────────────────────── Property value names ───────────────────────── + +# short name => long name, per property, in the order PropertyValueAliases.txt lists them. +$aliases = @{} +foreach ($line in Get-Content (Get-UcdFile 'PropertyValueAliases.txt')) { + if ($line -match '^\s*(lb|bc|ea|sc)\s*;\s*([^;#\s]+)\s*;\s*([^;#\s]+)') { + if (-not $aliases.ContainsKey($Matches[1])) { $aliases[$Matches[1]] = [ordered]@{} } + $aliases[$Matches[1]][$Matches[2]] = $Matches[3] + } +} + +<# Values of one property for every code point. The "@missing" defaults come first, in file order (later lines are + more specific), then the data lines. $names maps every name the file uses (short or long) to a value index. #> +function Read-Property([string]$file, [hashtable]$names) { + $values = [byte[]]::new(0x110000) + $missing = [System.Collections.Generic.List[object]]::new() + $data = [System.Collections.Generic.List[object]]::new() + foreach ($line in Get-Content $file) { + if ($line -match '^#\s*@missing:\s*([0-9A-F]+)\.\.([0-9A-F]+)\s*;\s*([^;#\s]+)') { + $missing.Add(@([Convert]::ToInt32($Matches[1], 16), [Convert]::ToInt32($Matches[2], 16), $Matches[3])) + } + elseif ($line -match '^([0-9A-F]+)(?:\.\.([0-9A-F]+))?\s*;\s*([^;#\s]+)') { + $first = [Convert]::ToInt32($Matches[1], 16) + $last = if ($Matches[2]) { [Convert]::ToInt32($Matches[2], 16) } else { $first } + $data.Add(@($first, $last, $Matches[3])) + } + } + if ($missing.Count -eq 0) { throw "$file has no @missing line." } + foreach ($range in @($missing) + @($data)) { + $name = $range[2] + if (-not $names.ContainsKey($name)) { throw "$file uses the unknown value '$name'." } + [Array]::Fill[byte]($values, [byte]$names[$name], $range[0], $range[1] - $range[0] + 1) + } + return , $values +} + +<# The values as ranges: (start, value) where the value changes. #> +function Get-Ranges([byte[]]$values) { + $starts = [System.Collections.Generic.List[int]]::new() + $result = [System.Collections.Generic.List[byte]]::new() + $previous = -1 + for ($i = 0; $i -lt $values.Length; $i++) { + if ($values[$i] -ne $previous) { + $starts.Add($i) + $result.Add($values[$i]) + $previous = $values[$i] + } + } + return , @($starts, $result) +} + +<# A C# enum member name from a UCD long name: "Caucasian_Albanian" => "CaucasianAlbanian". #> +function ConvertTo-Member([string]$name) { + return (($name -split '_') | ForEach-Object { $_.Substring(0, 1).ToUpperInvariant() + $_.Substring(1) }) -join '' +} + +# ───────────────────────── Writing ───────────────────────── + +$out = [System.Text.StringBuilder]::new() +function Line([string]$text = '') { [void]$out.Append($text).Append("`n") } + +<# One property: the enum (members in $order, as short or long names) and its range table. #> +function Write-Property([string]$enum, [string]$summary, [string]$table, [string]$file, [string]$property, + [string[]]$order, [switch]$LongMembers) { + $names = @{} + $long = $aliases[$property] + for ($i = 0; $i -lt $order.Count; $i++) { + $short = $order[$i] + $names[$short] = $i + $names[$long[$short]] = $i + } + $values = Read-Property $file $names + $ranges = Get-Ranges $values + Write-Host ("{0,-16} {1,3} values, {2,5} ranges" -f $enum, $order.Count, $ranges[0].Count) + + Line "/// $summary" + Line "internal enum $enum : byte" + Line '{' + for ($i = 0; $i -lt $order.Count; $i++) { + $short = $order[$i] + if ($LongMembers) { + Line " /// $short" + Line " $(ConvertTo-Member $long[$short])," + } + else { + Line " /// $($long[$short] -replace '_', ' ')" + Line " $short," + } + if ($i -lt $order.Count - 1) { Line } + } + Line '}' + Line + + $script:tables += , @($table, $enum, $ranges) +} + +$script:tables = @() +$lb = $aliases['lb'].Keys | Where-Object { $_ -ne 'XX' } +$bc = $aliases['bc'].Keys | Where-Object { $_ -ne 'L' } +$ea = $aliases['ea'].Keys | Where-Object { $_ -ne 'N' } +$sc = $aliases['sc'].Keys | Where-Object { $_ -notin 'Zzzz', 'Zyyy', 'Zinh' } + +Line '// ' +Line "// Generated by build/generate-unicode-tables.ps1 from the Unicode Character Database $Version (LineBreak.txt," +Line '// DerivedBidiClass.txt, Scripts.txt, EastAsianWidth.txt and PropertyValueAliases.txt). Do not edit: run the' +Line '// script. The data is Copyright (c) Unicode, Inc., under the Unicode License v3 (see THIRD-PARTY-NOTICES.md).' +Line '// ' +Line +Line 'namespace Filee.Engines.Typesetting;' +Line + +Write-Property 'LineBreakClass' 'Line_Break property values (UAX #14), by their short names.' 'LineBreak' ` +(Get-UcdFile 'LineBreak.txt') 'lb' (@('XX') + $lb) +Write-Property 'BidiClass' 'Bidi_Class property values (UAX #9), by their short names.' 'Bidi' ` +(Get-UcdFile 'extracted/DerivedBidiClass.txt') 'bc' (@('L') + $bc) +Write-Property 'EastAsianWidth' 'East_Asian_Width property values (UAX #11).' 'EastAsianWidth' ` +(Get-UcdFile 'EastAsianWidth.txt') 'ea' (@('N') + $ea) -LongMembers +Write-Property 'UnicodeScript' 'Script property values (UAX #24); the summary is the ISO 15924 code.' 'Script' ` +(Get-UcdFile 'Scripts.txt') 'sc' (@('Zzzz', 'Zyyy', 'Zinh') + $sc) -LongMembers + +Line 'internal static partial class UnicodeData' +Line '{' +Line ' /// Version of the Unicode Character Database the tables were generated from.' +Line " public const string Version = `"$Version`";" +Line +Line ' /// ISO 15924 code of each , by its value.' +$codes = @(@('Zzzz', 'Zyyy', 'Zinh') + $sc | ForEach-Object { "`"$_`"" }) +Line ' private static readonly string[] ScriptCodes =' +Line ' [' +for ($i = 0; $i -lt $codes.Count; $i += 14) { + Line (' ' + ($codes[$i..([Math]::Min($i + 13, $codes.Count - 1))] -join ', ') + ',') +} +Line ' ];' +foreach ($entry in $script:tables) { + $table, $enum, $ranges = $entry + $starts, $values = $ranges + Line + Line " /// First code point of each range of equal values, ascending." + Line " private static ReadOnlySpan ${table}Starts =>" + Line ' [' + for ($i = 0; $i -lt $starts.Count; $i += 10) { + $count = [Math]::Min(10, $starts.Count - $i) + Line (' ' + (($starts.GetRange($i, $count) | ForEach-Object { '0x{0:X4}' -f $_ }) -join ', ') + ',') + } + Line ' ];' + Line + Line " /// The value of each range in ." + Line " private static ReadOnlySpan ${table}Values =>" + Line ' [' + for ($i = 0; $i -lt $values.Count; $i += 24) { + $count = [Math]::Min(24, $values.Count - $i) + Line (' ' + (($values.GetRange($i, $count) | ForEach-Object { "$_" }) -join ', ') + ',') + } + Line ' ];' +} +Line '}' + +$Output = [IO.Path]::GetFullPath($Output) +[IO.File]::WriteAllText($Output, $out.ToString(), [Text.UTF8Encoding]::new($false)) +Write-Host "Wrote $Output" diff --git a/docs/ENGINES.md b/docs/ENGINES.md index 2cc1db4f..b8bd9261 100644 --- a/docs/ENGINES.md +++ b/docs/ENGINES.md @@ -602,12 +602,15 @@ document). If tagged PDF (PDF/UA) becomes a goal, it would need post-processing ### Still open -- Variable fonts: the subset is the default instance. A non-default instance (a weight the font resolver picks from a - variable font) needs an instancer that applies gvar / HVAR deltas before subsetting; until then such a typeface - would go to Skia whole (Type 3). -- CFF2 outlines can't be converted or subset yet (Type 3 through Skia). +- Variable fonts: the subset is the default instance. A non-default instance needs an instancer that applies gvar / + HVAR deltas before shaping and subsetting. The font resolver doesn't pick one: it uses a variable font at its default + instance and reports a missing weight as synthetic (see [fonts, shaping and Unicode + tables](#filee-renderer-fonts-shaping-and-unicode-tables)). +- CFF2 outlines can't be converted or subset yet (Type 3 through Skia). The font resolver uses a CFF2 font only when no + other font for its script is installed. - HarfBuzzSharp's `Blob.FromStream` let HarfBuzz read freed memory in the spike's shaper (wrong glyphs, then a - crash); the shaper should hand HarfBuzz font data in memory it owns, as `Vector/GlyphOutlines.cs` does. + crash). The layout engine's shaper (`Typesetting/Shaper.cs`) hands HarfBuzz `PageFont.Data` pinned for as long as + HarfBuzz's face lives. The spike also fixed CID-keyed CFF fonts whose Top DICT and font dictionary both carry a units-per-em scale (Top DICT 1/2048, font dictionary 1/1000, as in Pretendard): the two FontMatrix scales were multiplied, so OTF → TTF and EOT gave @@ -619,6 +622,68 @@ trips); `tests/Filee.Engines.Tests/PdfPageWriterTests.cs` builds the page above PDFium picture, with each installed Korean font (Malgun Gothic, Noto Sans CJK, Noto Sans KR VF; skipped when none is installed), plus page sizes, links and the outline. +## Filee renderer: fonts, shaping and Unicode tables + +The text services of Filee's own document renderer (GitHub Project 4, item *Layout engine: font resolver, HarfBuzz +shaper and Unicode tables*), in `src/Filee.Engines/Typesetting`. No converter uses them yet; paragraph layout, CJK +typography and the bidirectional algorithm build on them. They use only what Filee already runs in-process: SkiaSharp's +font manager, HarfBuzzSharp and .NET. + +- **Font slots** (`FontSlots.cs`): which of a run's fonts draws a character. 한글's seven language slots (한글, 영문, 한자, + 일어, 외국어, 기호, 사용자) go by script: ASCII, Latin-1 and the spacing modifiers to 영문, quotes, dashes, CJK + punctuation, fullwidth forms and other symbols to 기호, private use to 사용자, Greek, Cyrillic, Arabic, Thai, ... to + 외국어. Word's four run fonts: ASCII to `ascii`, right-to-left, Brahmic and Southeast Asian scripts to `cs`, CJK + scripts and blocks to `eastAsia`, the rest to `hAnsi`; with `w:hint="eastAsia"` ambiguous-width characters and + private use go to `eastAsia`. Word's finer rule for Latin-1 letters in Chinese text (hint plus East Asian language) + isn't made. A grapheme cluster takes the slot of its first character, so marks stay with their base. +- **Font resolver** (`FontResolver.cs`, `FontNames.cs`, `SystemFonts.cs`): installed fonts through `SKFontManager` + (DirectWrite, CoreText, fontconfig); nothing is downloaded or shipped. + - Names: a family is found by its name, by its name without trailing style words ("Pretendard Medium" is weight + 500), and by other names: Korean ↔ English names (함초롬바탕 ↔ HCR Batang, 맑은 고딕 ↔ Malgun Gothic, the HY + fonts, Nanum fonts; checked against the fonts' name tables), the fonts built into 한글 (한양신명조, 한양중고딕, ...) + → the HY TrueType fonts of the same design, and metric-compatible replacements (Arial → Liberation Sans, Calibri → + Carlito, ...). Generic families (serif, sans-serif, monospace) get the language's default fonts. + - Style: the face nearest the requested weight and slant by the CSS matching rules. A bold or italic the family + lacks is reported as synthetic (`ResolvedFont.SyntheticBold` / `SyntheticItalic`) for the drawing to fake. + - Missing fonts and the document-wide rules come from `RhwpFonts.cs`, now shared (`FontSubstitution.cs`): a missing + font is replaced by an installed one of the script its name points to (else the text's language) and the same + style (serif or sans); `Plan` swaps Korean default fonts in Chinese or Japanese text (#64) and gives text without + Hangul, Han or kana one font that has all of it. rhwp keeps using the same rules through `RhwpFonts`. + - Per-cluster fallback: a run is split into pieces each drawn by one font. A grapheme cluster keeps the run's font + when it has every character of it, else takes a font already used in the run, a font for its script (a list per + script: Nirmala UI, Leelawadee UI, Noto, Lohit, ... ; Han by the text's language), then the platform's own + fallback (`SKFontManager.MatchCharacter`). Punctuation and spaces stay with the font of the text around them. + - Variable fonts are used at their default instance: HarfBuzzSharp 8.3 has no binding for variation coordinates, the + PDF subset is the default instance, and Filee has no instancer. DirectWrite's named instances of one variable + font count as one face, with the weight the font's own OS/2 table gives; a heavier weight is synthetic bold. + - CFF2 fonts (Source Han VF and the like) are used only when no other font for their script is installed: the + subsetter can't take them, so Skia would embed them as Type 3 fonts (still searchable, not subset). +- **Shaper** (`Shaper.cs`): HarfBuzz shaping generalised from `Vector/GlyphOutlines.cs`. A run is shaped with up to + five characters of the paragraph on either side as context (HarfBuzz's own context length), so a word split into two + runs still joins and reorders as one. Results are in font units, cached by font, features, script, direction, + language and the run's text with its context, and carry glyph IDs, advances, offsets, HarfBuzz's cluster per glyph + (monotone graphemes, visual order) and its unsafe-to-break flags for the line breaker. `ShapedRun.ToGlyphRun` makes + the page model's `GlyphRun` (`PdfPageWriterTests` now builds its page with it). `Shaper.Metrics` gives ascender, + descender, line gap, Win metrics, x-height, cap height, underline, strikeout and super / subscript metrics. +- **Unicode tables** (`UnicodeData.cs`, `UnicodeData.Generated.cs`): Line_Break, Bidi_Class, Script and + East_Asian_Width for every code point, as range tables searched by binary search; unassigned code points get the + UCD's defaults (`@missing` lines: ID for the CJK blocks, R and AL for the right-to-left blocks, ...). + `build/generate-unicode-tables.ps1` downloads the pinned UCD files from unicode.org into `build/.cache` (SHA-256 + checked, not committed) and writes the C# file, which is committed (Unicode License v3, in THIRD-PARTY-NOTICES.md). + - Version: **Unicode 16.0.0**, the version .NET 10 uses for `CharUnicodeInfo`, `Rune` and `StringInfo` (checked by + a test over every code point), so the layout engine's properties and .NET's grapheme clusters and categories agree. + HarfBuzz 8.3 inside HarfBuzzSharp uses Unicode 15.1 for its own normalization and mark handling; the scripts added + in 16.0 (Garay, Todhri, Sunuwar, ...) shape with HarfBuzz's default shaper until HarfBuzzSharp moves on. Change the + version together with the target framework: update `$Version` and the hashes and run the script. + +Tests: `FontSlotsTests`, `UnicodeDataTests`, `FontResolverTests` (a fake font list like `RhwpFontsTests`, plus one +check of the installed Malgun Gothic) and `ShaperTests`. The shaper is checked three ways for Arabic, Devanagari, +Gujarati, Thai and Hangul jamo: against HarfBuzz itself (the text shaped the plain way, one buffer with guessed +properties) with every installed font of the script, against recorded HarfBuzz output for the pinned Noto fonts of +`build/fetch-fonts.ps1` (CI fetches them; there is no pinned Gujarati font), and by what each script needs whatever the +font (joining forms and lam-alef, vowel signs drawn before their consonant, conjuncts, the SARA AM split, jamo composed +into precomposed syllables, a dotted circle only for a broken cluster). + ## DOCX writer The counterpart of the HWPX writer: `Hwp/Hwpx/Docx/DocxWriter*.cs` writes WordprocessingML by hand from the same diff --git a/src/Filee.Engines/Hwp/RhwpFonts.cs b/src/Filee.Engines/Hwp/RhwpFonts.cs index 9b3c9222..8a13eff6 100644 --- a/src/Filee.Engines/Hwp/RhwpFonts.cs +++ b/src/Filee.Engines/Hwp/RhwpFonts.cs @@ -1,59 +1,26 @@ // Fonts for rhwp, chosen from the fonts installed on the computer (RhwpInput writes them into a copy of the HWPX). // Nothing is downloaded or shipped. // -// - Language: documents Filee writes (Markdown, text, HTML → HWPX) name Korean fonts in every slot. In a Chinese or -// Japanese document those drew Korean-style Hanja and, for characters they lack, a different fallback font per -// character (issue #64). Without Hangul in the text, Korean fonts are swapped for an installed Chinese (Simplified -// or Traditional) or Japanese font of the same style. -// - Coverage: rhwp draws a character with the font of its slot when that font has it, else with a fallback of its own -// (Arabic came out in a calligraphic font). In text without Hangul, Han or kana, a slot's font is swapped for an -// installed one of the same style that has every character of the text the old one had, and more of the rest: -// Russian, Greek, Thai or Arabic text gets one consistent font. +// The language and coverage rules are shared with the layout engine's font resolver (Typesetting/FontSubstitution.cs): +// Korean default fonts in Chinese or Japanese text (issue #64), one consistent font for text without Hangul, Han or +// kana. What is rhwp's own stays here: +// // - CFF2: variable fonts with CFF2 outlines (Source Han Serif VF, "思源宋体 VF") fail rhwp's font subsetter and the -// page came out blank (issue #61); they are always swapped. +// page came out blank (issue #61); they are not usable and are always swapped. +// - The lookup through Skia's font manager, cached per name for the HWPX copies rhwp gets. using System.Collections.Concurrent; -using System.Globalization; -using System.Text; +using Filee.Engines.Typesetting; using SkiaSharp; namespace Filee.Engines.Hwp; /// An installed font: whether rhwp can embed it, which characters it has and how wide they are. /// Advance width of a character in em, null when the font lacks it (or is not measured). -internal sealed record InstalledFont(bool Usable, Func Has, Func? Advance = null); +internal sealed record InstalledFont(bool Usable, Func Has, Func? Advance = null) : FontCoverage(Usable, Has); internal static class RhwpFonts { - private static readonly string[] Styles = - ["Thin", "ExtraLight", "Light", "Normal", "Regular", "Medium", "SemiBold", "Bold", "Heavy", "Black", "ExtraBold"]; - - /// Installed fonts in order of preference, by script, serif first then sans. - private static readonly Dictionary Candidates = new() - { - ["zh-Hans"] = (["SimSun", "NSimSun", "Noto Serif SC", "Songti SC", "Noto Serif CJK SC"], - ["Microsoft YaHei", "DengXian", "Noto Sans SC", "PingFang SC", "Noto Sans CJK SC", "SimHei"]), - ["zh-Hant"] = (["PMingLiU", "MingLiU", "Noto Serif TC", "Songti TC", "Noto Serif CJK TC"], - ["Microsoft JhengHei", "Noto Sans TC", "PingFang TC", "Noto Sans CJK TC"]), - ["ja"] = (["Yu Mincho", "MS Mincho", "Noto Serif JP", "Hiragino Mincho ProN", "Noto Serif CJK JP"], - ["Yu Gothic", "Meiryo", "MS Gothic", "Noto Sans JP", "Hiragino Sans", "Noto Sans CJK JP"]), - ["ko"] = (["Batang", "HCR Batang", "Noto Serif KR", "AppleMyungjo", "Noto Serif CJK KR"], - ["Malgun Gothic", "Noto Sans KR", "Apple SD Gothic Neo", "NanumGothic", "Noto Sans CJK KR"]), - // Wide coverage first (Tahoma and Times New Roman also have Greek, Cyrillic, Arabic, Hebrew; Tahoma Thai). - [""] = (["Times New Roman", "Noto Serif", "DejaVu Serif", "Georgia", "Liberation Serif"], - ["Segoe UI", "Tahoma", "Arial", "Noto Sans", "DejaVu Sans", "Helvetica", "Liberation Sans"]), - }; - - /// Scripts whose fonts rhwp falls back to on its own and the computer may have better ones for. - private static readonly string[] WideFonts = ["Leelawadee UI", "Nirmala UI", "Microsoft Sans Serif", "Arial Unicode MS"]; - - /// - /// Simplified and Traditional forms of common characters: which form a Chinese text uses. Forms Japanese writes - /// the same way (国, 学, 会, 点, …) are left out. - /// - private const string SimplifiedOnly = "这们说为时个对发经过还进动东车长门间问见现实从开关书无让应该话样"; - private const string TraditionalOnly = "這們說為時個對發經過還進動東車長門間問見現實從開關書無讓應該話樣"; - private static readonly ConcurrentDictionary Fonts = new(StringComparer.OrdinalIgnoreCase); /// Forgets the fonts looked up so far (). @@ -62,129 +29,13 @@ internal static class RhwpFonts /// Rechecks one family after the user may have installed it during this session. internal static void Forget(string face) => Fonts.TryRemove(face, out _); - /// Replacements by (slot, face): the language, coverage and CFF2 rules above. + /// Replacements by (slot, face): the language, coverage and CFF2 rules (see ). internal static Dictionary<(string Slot, string Face), string> Plan( - IReadOnlyList<(string Slot, string Face)> slots, string text, Func lookup) - { - var characters = Characters(text); - var language = Language(text); - var result = new Dictionary<(string, string), string>(); - foreach (var (slot, face) in slots.Distinct()) - { - if (face.Length == 0) - continue; - var serif = IsSerif(face); - var chosen = face; - var current = lookup(face); - - // Korean fonts in a Chinese or Japanese text; a font rhwp cannot embed. - if (language is not ("ko" or "") && Script(face) == "ko" || current is { Usable: false }) - { - var script = current is { Usable: false } && Script(face) is { Length: > 0 } own ? own : language; - if (Preferred(script, serif).FirstOrDefault(name => lookup(name) is { Usable: true }) is { } replacement) - chosen = replacement; - } - - // Text without Hangul, Han or kana (Russian, Greek, Thai, Arabic, …): a font of the same style with every - // character the chosen one has, and more of the others. In Korean, Chinese and Japanese text the language - // font stays, and rhwp's own fallback draws the odd foreign word as before. - var mine = language != "" || slot is "HANJA" or "JAPANESE" - ? [] - : characters.Where(c => !IsHan(c) && !IsKana(c) && (slot == "HANGUL" || !IsHangul(c))).ToList(); - if (mine.Count > 0 && lookup(chosen) is { Usable: true } font) - { - var covered = mine.Where(c => font.Has(c)).ToList(); - if (covered.Count < mine.Count) - { - var best = chosen; - var bestCount = covered.Count; - var sameStyle = (serif ? Styled(language).Serif.Concat(Styled("").Serif) : Styled(language).Sans.Concat(Styled("").Sans).Concat(WideFonts)) - .Distinct(StringComparer.OrdinalIgnoreCase); - foreach (var name in sameStyle) - { - if (lookup(name) is not { Usable: true } other || !covered.All(other.Has)) - continue; - var count = mine.Count(other.Has); - if (count == mine.Count) - { - (best, bestCount) = (name, count); // the first one in order that has everything - break; - } - if (count > bestCount) - (best, bestCount) = (name, count); - } - chosen = best; - } - } - - if (!string.Equals(chosen, face, StringComparison.Ordinal)) - result[(slot, face)] = chosen; - } - return result; - } - - private static IEnumerable Preferred(string script, bool serif) - { - var (serifs, sans) = Styled(script); - return serif ? serifs.Concat(sans) : sans.Concat(serifs); - } - - private static (string[] Serif, string[] Sans) Styled(string script) => - Candidates.TryGetValue(script, out var c) ? c : Candidates[""]; - - private static bool IsHangul(int c) => c is >= 0xAC00 and <= 0xD7A3 or >= 0x1100 and <= 0x11FF or >= 0x3130 and <= 0x318F; + IReadOnlyList<(string Slot, string Face)> slots, string text, Func lookup) => + FontSubstitution.Plan(slots, text, lookup); - private static bool IsKana(int c) => c is >= 0x3040 and <= 0x30FF; - - private static bool IsHan(int c) => c is >= 0x4E00 and <= 0x9FFF or >= 0x3400 and <= 0x4DBF or >= 0x20000 and <= 0x2FFFF; - - /// - /// The language the Han characters of a text are in: Korean with Hangul, Japanese with kana, else Simplified or - /// Traditional Chinese by common characters; "" without Han characters. - /// - internal static string Language(string text) - { - int hangul = 0, kana = 0, han = 0, simplified = 0, traditional = 0; - foreach (var c in text) - { - if (c is >= '가' and <= '힣' or >= 'ᄀ' and <= 'ᇿ' or >= '㄰' and <= '㆏') - hangul++; - else if (c is >= '぀' and <= 'ヿ') - kana++; - else if (c is >= '一' and <= '鿿' or >= '㐀' and <= '䶿') - { - han++; - if (SimplifiedOnly.Contains(c)) - simplified++; - else if (TraditionalOnly.Contains(c)) - traditional++; - } - } - if (hangul > 0) - return "ko"; - // Japanese prose is mostly kana; a Chinese text quoting a Japanese title is not. - if (kana > 0 && (simplified + traditional == 0 || kana * 3 >= han)) - return "ja"; - if (han > 0) - return traditional > simplified ? "zh-Hant" : "zh-Hans"; - return ""; - } - - /// Distinct characters of the text that need a glyph (no spaces or controls), at most 4000. - private static List Characters(string text) - { - var set = new HashSet(); - foreach (var rune in text.EnumerateRunes()) - { - if (set.Count >= 4000) - break; - if (Rune.GetUnicodeCategory(rune) is not (UnicodeCategory.SpaceSeparator or UnicodeCategory.Control - or UnicodeCategory.Format or UnicodeCategory.LineSeparator or UnicodeCategory.ParagraphSeparator - or UnicodeCategory.NonSpacingMark or UnicodeCategory.PrivateUse or UnicodeCategory.OtherNotAssigned)) - set.Add(rune.Value); - } - return set.ToList(); - } + /// + internal static string Language(string text) => FontSubstitution.Language(text); /// /// An installed font by name, or null. "思源宋体 VF Medium" names a style of "思源宋体 VF", so trailing style words @@ -192,7 +43,7 @@ or UnicodeCategory.Format or UnicodeCategory.LineSeparator or UnicodeCategory.Pa /// internal static InstalledFont? Find(string face) => Fonts.GetOrAdd(face, static name => { - foreach (var candidate in Names(name)) + foreach (var (candidate, _) in FontSubstitution.Names(name)) { var typeface = SKFontManager.Default.MatchFamily(candidate); // Windows matches localized names ("思源宋体 VF") and returns nothing for unknown ones; fontconfig always @@ -220,40 +71,11 @@ or UnicodeCategory.Format or UnicodeCategory.LineSeparator or UnicodeCategory.Pa /// False when the font has CFF2 outlines (variable OpenType fonts made from PostScript outlines). internal static bool IsUsable(SKTypeface typeface) => !typeface.GetTableTags().Contains(Tag("CFF2")); - private static IEnumerable Names(string face) - { - yield return face; - var words = face.Split(' ', StringSplitOptions.RemoveEmptyEntries).ToList(); - while (words.Count > 1 && Styles.Contains(words[^1], StringComparer.OrdinalIgnoreCase)) - { - words.RemoveAt(words.Count - 1); - yield return string.Join(' ', words); - } - } - - /// The script a font name points to: "思源宋体" and "SC" are Simplified Chinese, "명조" Korean, …. - internal static string Script(string face) - { - bool Has(params string[] parts) => parts.Any(p => face.Contains(p, StringComparison.OrdinalIgnoreCase)); - bool Word(params string[] words) => face.Split(' ', '-', '_').Any(w => words.Contains(w, StringComparer.OrdinalIgnoreCase)); - if (Word("SC", "CN", "GB") || Has("简", "宋体", "黑体", "仿宋", "楷体")) - return "zh-Hans"; - if (Word("TC", "HK", "TW") || Has("繁", "明體", "黑體", "宋體")) - return "zh-Hant"; - if (Word("JP", "J") || Has("明朝", "ゴシック", "游")) - return "ja"; - if (Word("KR", "K") || face.Any(c => c is >= '가' and <= '힣') - || Candidates["ko"].Serif.Concat(Candidates["ko"].Sans).Contains(face, StringComparer.OrdinalIgnoreCase)) - return "ko"; - if (face.Any(c => c is >= '一' and <= '鿿')) - return "zh-Hans"; - return ""; - } + /// + internal static string Script(string face) => FontSubstitution.Script(face); - /// Serif (Song/Ming/Mincho/Batang) rather than sans (Hei/Gothic/Dotum). - internal static bool IsSerif(string face) => - new[] { "Serif", "Song", "Ming", "Mincho", "Myeongjo", "Myungjo", "Batang", "Times", "宋", "明", "바탕", "명조", "仿宋", "楷" } - .Any(p => face.Contains(p, StringComparison.OrdinalIgnoreCase)); + /// + internal static bool IsSerif(string face) => FontSubstitution.IsSerif(face); private static uint Tag(string tag) => (uint)tag[0] << 24 | (uint)tag[1] << 16 | (uint)tag[2] << 8 | tag[3]; } diff --git a/src/Filee.Engines/Infrastructure/EngineCaches.cs b/src/Filee.Engines/Infrastructure/EngineCaches.cs index 2045f502..3c42f8d2 100644 --- a/src/Filee.Engines/Infrastructure/EngineCaches.cs +++ b/src/Filee.Engines/Infrastructure/EngineCaches.cs @@ -1,9 +1,11 @@ -// Caches engines keep between conversions (typefaces looked up for HWP and CAD text). The app releases them while no -// conversion runs, so a Filee waiting in the tray gives the memory back. Skia's own caches stay: they are shared with -// the UI's render thread, and purging them while it draws crashed in native code (SKGraphics.PurgeAllCaches). +// Caches engines keep between conversions (typefaces looked up for HWP, CAD and layout text). The app releases them +// while no conversion runs, so a Filee waiting in the tray gives the memory back. Skia's own caches stay: they are +// shared with the UI's render thread, and purging them while it draws crashed in native code +// (SKGraphics.PurgeAllCaches). using Filee.Engines.Cad; using Filee.Engines.Hwp; +using Filee.Engines.Typesetting; namespace Filee.Engines.Infrastructure; @@ -19,5 +21,6 @@ public static void Release() { RhwpFonts.ReleaseCache(); CadFonts.ReleaseCache(); + SystemFonts.ReleaseCache(); } } diff --git a/src/Filee.Engines/Typesetting/FontCollection.cs b/src/Filee.Engines/Typesetting/FontCollection.cs new file mode 100644 index 00000000..6f5d176a --- /dev/null +++ b/src/Filee.Engines/Typesetting/FontCollection.cs @@ -0,0 +1,84 @@ +// The fonts the layout engine can draw with, as FontResolver sees them: families of faces with their style, the +// characters they have and the font file behind them. SystemFonts lists the installed fonts; tests pass a list of +// their own. + +using Filee.Engines.Rendering; + +namespace Filee.Engines.Typesetting; + +/// A collection of fonts: the installed ones () or a fixed list. +internal interface IFontCollection +{ + /// + /// The faces of the family with exactly this name (any name the system knows it by, such as a localized one); + /// empty when no such family is installed. + /// + IReadOnlyList Family(string name); + + /// + /// A face the system would draw with when none of the named fonts has it (the + /// platform's own fallback), or null. + /// + /// BCP 47 language of the text ("ja", "zh-Hant"), which picks between CJK fonts. + FontFace? ForCharacter(int codePoint, FontStyle style, string? language); +} + +/// A requested style: weight 100–900 (400 regular, 700 bold) and italic. +internal readonly record struct FontStyle(int Weight = 400, bool Italic = false) +{ + public static readonly FontStyle Regular = new(); + + public bool Bold => Weight >= 600; + + /// A style from a bold / italic pair, as documents store it. + public static FontStyle From(bool bold, bool italic) => new(bold ? 700 : 400, italic); +} + +/// One face of a font: name, style, coverage and the file it comes from. +internal sealed class FontFace +{ + private readonly Func _has; + private readonly Lazy? _font; + + /// Whether the face has a glyph for a code point. + /// Reads the font file (null for faces that are only matched, in tests). + public FontFace(string family, int weight, bool italic, Func has, Func? open = null) + { + Family = family; + Weight = weight; + Italic = italic; + _has = has; + _font = open is null ? null : new Lazy(open, LazyThreadSafetyMode.ExecutionAndPublication); + } + + /// The family name the collection lists the face under. + public string Family { get; } + + /// Weight class (OS/2 usWeightClass); a variable font's is that of its default instance. + public int Weight { get; } + + public bool Italic { get; } + + /// + /// A variable font (fvar). It is shaped, subset and drawn at its default instance: other instances need an + /// instancer that applies the variation deltas, which Filee doesn't have. A weight the default instance doesn't + /// have is synthesized like for any family without that weight. + /// + public bool Variable { get; init; } + + /// + /// CFF2 (variable PostScript) outlines: the PDF writer can't subset them, so Skia embeds the font as a Type 3 font + /// (still searchable). The resolver uses such a face only when no other font for its script is installed. + /// + public bool Cff2 { get; init; } + + /// True when the face has a glyph for . + public bool Has(int codePoint) => _has(codePoint); + + /// The font file, read on first use. + /// The face has no font file (made only for matching). + /// The font file can't be read. + public PageFont Font => _font?.Value ?? throw new InvalidOperationException($"{Family} has no font file."); + + public override string ToString() => $"{Family} {Weight}{(Italic ? " italic" : "")}"; +} diff --git a/src/Filee.Engines/Typesetting/FontNames.cs b/src/Filee.Engines/Typesetting/FontNames.cs new file mode 100644 index 00000000..6a29a4ee --- /dev/null +++ b/src/Filee.Engines/Typesetting/FontNames.cs @@ -0,0 +1,92 @@ +// Other names for the fonts documents name. 한글 documents name Korean fonts by their Korean family names +// ("함초롬바탕"), Word documents usually by the English ones ("Malgun Gothic"); both are in the fonts' name tables, but +// only DirectWrite matches either: fontconfig and CoreText may know just one. Older 한글 documents name the fonts +// built into 한글 (HFT: 한양신명조, 한양중고딕, ...), which are Hanyang's designs that Windows and Office install as +// TrueType fonts (HY신명조, HY중고딕, ...). And documents from other systems name fonts that have metric-compatible +// open replacements (Arial → Liberation Sans), which keep line breaks where they were. + +namespace Filee.Engines.Typesetting; + +/// Alternative names for font families. +internal static class FontNames +{ + /// Korean and English family names of the same fonts, as their name tables list them. + private static readonly (string Korean, string English)[] KoreanNames = + [ + ("함초롬바탕", "HCR Batang"), ("함초롬돋움", "HCR Dotum"), ("맑은 고딕", "Malgun Gothic"), ("한컴 고딕", "Hancom Gothic"), + ("바탕", "Batang"), ("바탕체", "BatangChe"), ("궁서", "Gungsuh"), ("궁서체", "GungsuhChe"), + ("굴림", "Gulim"), ("굴림체", "GulimChe"), ("돋움", "Dotum"), ("돋움체", "DotumChe"), + ("HY신명조", "HYSinMyeongJo-Medium"), ("HY중고딕", "HYGothic-Medium"), ("HY견명조", "HYMyeongJo-Extra"), + ("HY견고딕", "HYGothic-Extra"), ("HY궁서B", "HYGungSo-Bold"), ("HY그래픽M", "HYGraphic-Medium"), + ("HY헤드라인M", "HYHeadLine-Medium"), ("HY엽서L", "HYPost-Light"), ("HY엽서M", "HYPost-Medium"), + ("HY얕은샘물M", "HYShortSamul-Medium"), ("HY목각파임B", "HYPMokGak-Bold"), + ("나눔고딕", "NanumGothic"), ("나눔명조", "NanumMyeongjo"), ("나눔바른고딕", "NanumBarunGothic"), + ("나눔바른펜", "NanumBarunpen"), ("나눔스퀘어", "NanumSquare"), ("나눔스퀘어라운드", "NanumSquareRound"), + ("나눔손글씨 붓", "Nanum Brush Script"), ("나눔손글씨 펜", "Nanum Pen Script"), + ("나눔고딕 에코", "NanumGothic Eco"), ("나눔명조 에코", "NanumMyeongjo Eco"), + ]; + + /// Fonts built into 한글 (HFT) and the TrueType fonts of the same design. + private static readonly Dictionary HancomBuiltIn = new(StringComparer.OrdinalIgnoreCase) + { + ["한양신명조"] = "HY신명조", + ["신명조"] = "HY신명조", + ["한양중고딕"] = "HY중고딕", + ["중고딕"] = "HY중고딕", + ["한양견명조"] = "HY견명조", + ["견명조"] = "HY견명조", + ["한양견고딕"] = "HY견고딕", + ["견고딕"] = "HY견고딕", + ["한양궁서"] = "HY궁서B", + ["한양그래픽"] = "HY그래픽M", + ["한양헤드라인"] = "HY헤드라인M", + }; + + /// Metric-compatible replacements: same advance widths, so text breaks into the same lines. + private static readonly Dictionary MetricCompatible = new(StringComparer.OrdinalIgnoreCase) + { + ["Arial"] = ["Liberation Sans", "Arimo"], + ["Helvetica"] = ["Arial", "Liberation Sans", "Arimo"], + ["Times New Roman"] = ["Liberation Serif", "Tinos"], + ["Times"] = ["Times New Roman", "Liberation Serif", "Tinos"], + ["Courier New"] = ["Liberation Mono", "Cousine"], + ["Courier"] = ["Courier New", "Liberation Mono", "Cousine"], + ["Calibri"] = ["Carlito"], + ["Calibri Light"] = ["Carlito"], + ["Cambria"] = ["Caladea"], + }; + + private static readonly Dictionary Translations = KoreanNames + .SelectMany(n => new[] { (n.Korean, n.English), (n.English, n.Korean) }) + .ToDictionary(p => p.Item1, p => p.Item2, StringComparer.OrdinalIgnoreCase); + + /// + /// Other names to look a family up by, best first: its name in the other language, the TrueType font of a 한글 + /// built-in font (by both names), then metric-compatible replacements. + /// + public static IEnumerable Alternatives(string family) + { + if (Translations.TryGetValue(family, out var translated)) + yield return translated; + if (HancomBuiltIn.TryGetValue(family, out var builtIn)) + { + yield return builtIn; + if (Translations.TryGetValue(builtIn, out var english)) + yield return english; + } + if (MetricCompatible.TryGetValue(family, out var compatible)) + { + foreach (var name in compatible) + yield return name; + } + } + + /// The generic families of CSS and HTML: serif, sans-serif, monospace and the rest. + public static bool IsGeneric(string family) => family.Trim().ToLowerInvariant() is "serif" or "sans-serif" or "sans serif" + or "monospace" or "cursive" or "fantasy" or "system-ui" or "ui-serif" or "ui-sans-serif" or "ui-monospace"; + + /// Monospaced fonts, for the generic "monospace" family, best first. + public static IReadOnlyList Monospace { get; } = + ["Consolas", "Cascadia Mono", "D2Coding", "Courier New", "Menlo", "SF Mono", "DejaVu Sans Mono", "Liberation Mono", + "Noto Sans Mono", "GulimChe"]; +} diff --git a/src/Filee.Engines/Typesetting/FontResolver.cs b/src/Filee.Engines/Typesetting/FontResolver.cs new file mode 100644 index 00000000..98a03d44 --- /dev/null +++ b/src/Filee.Engines/Typesetting/FontResolver.cs @@ -0,0 +1,312 @@ +// Fonts for the layout engine, from the fonts installed on the computer (IFontCollection; SystemFonts by default). +// Nothing is downloaded or shipped. +// +// - Names: a family is looked up by its name, by its name without trailing style words ("Pretendard Medium" is +// Pretendard at weight 500), and by its other names (FontNames: 함초롬바탕 ↔ HCR Batang, 한글's built-in 한양신명조 → +// HY신명조, metric-compatible replacements such as Arial → Liberation Sans). +// - Style: the face nearest the requested weight and slant, by the CSS font matching rules. What the family lacks +// (bold, italic) is reported as synthetic for the drawing to fake. +// - Missing fonts: a font of the same script (from its name, else the text's language) and style (serif or sans) is +// used instead, as RhwpFonts does for rhwp (FontSubstitution). Plan applies the document-wide language and coverage +// rules (Korean default fonts in Chinese or Japanese text, one consistent font for Russian or Arabic text). +// - Fallback: a run is split per grapheme cluster into pieces each drawn by one font: the run's own font when it has +// every character of the cluster, else a font for the cluster's script, else the platform's fallback. Punctuation +// and spaces stay with their neighbours' font when it has them. +// - Variable and CFF2 fonts: a variable font counts as its default instance (no instancer yet); a CFF2 font is used +// only when no other font for its script is installed, because the PDF writer can't subset it (FontFace). + +using System.Collections.Concurrent; +using System.Globalization; +using Filee.Engines.Rendering; + +namespace Filee.Engines.Typesetting; + +/// A font the layout draws with: an installed face, and what the drawing has to fake. +/// Bold was asked for and the family has no bold face (embolden the outlines). +/// Italic was asked for and the family has no italic face (slant the outlines). +internal sealed record ResolvedFont(FontFace Face, bool SyntheticBold = false, bool SyntheticItalic = false) +{ + /// The font file (read on first use). + public PageFont Font => Face.Font; + + public string Family => Face.Family; + + public override string ToString() => + $"{Face}{(SyntheticBold ? " +bold" : "")}{(SyntheticItalic ? " +italic" : "")}"; +} + +/// A piece of text drawn with one font. +/// UTF-16 index of the first character. +/// Length in UTF-16 code units. +internal readonly record struct FontRun(int Start, int Length, ResolvedFont Font) +{ + public int End => Start + Length; +} + +/// Finds installed fonts for the names documents give, and fonts for the characters those lack. +internal sealed class FontResolver +{ + /// Installed fonts for scripts the language candidates may lack, best first. + private static readonly Dictionary ScriptFonts = new() + { + [UnicodeScript.Arabic] = ["Segoe UI", "Arial", "Tahoma", "Times New Roman", "Noto Sans Arabic", "Noto Naskh Arabic", "Geeza Pro", "DejaVu Sans"], + [UnicodeScript.Hebrew] = ["Segoe UI", "Arial", "Times New Roman", "Noto Sans Hebrew", "Arial Hebrew", "DejaVu Sans"], + [UnicodeScript.Syriac] = ["Segoe UI Historic", "Estrangelo Edessa", "Noto Sans Syriac"], + [UnicodeScript.Thaana] = ["MV Boli", "Noto Sans Thaana"], + [UnicodeScript.Devanagari] = ["Nirmala UI", "Mangal", "Noto Sans Devanagari", "Kohinoor Devanagari", "Devanagari Sangam MN", "Lohit Devanagari"], + [UnicodeScript.Bengali] = ["Nirmala UI", "Vrinda", "Noto Sans Bengali", "Kohinoor Bangla", "Bangla Sangam MN", "Lohit Bengali"], + [UnicodeScript.Gurmukhi] = ["Nirmala UI", "Raavi", "Noto Sans Gurmukhi", "Gurmukhi Sangam MN", "Lohit Gurmukhi"], + [UnicodeScript.Gujarati] = ["Nirmala UI", "Shruti", "Noto Sans Gujarati", "Kohinoor Gujarati", "Gujarati Sangam MN", "Lohit Gujarati"], + [UnicodeScript.Oriya] = ["Nirmala UI", "Kalinga", "Noto Sans Oriya", "Oriya Sangam MN", "Lohit Odia"], + [UnicodeScript.Tamil] = ["Nirmala UI", "Latha", "Noto Sans Tamil", "Tamil Sangam MN", "Lohit Tamil"], + [UnicodeScript.Telugu] = ["Nirmala UI", "Gautami", "Noto Sans Telugu", "Kohinoor Telugu", "Telugu Sangam MN", "Lohit Telugu"], + [UnicodeScript.Kannada] = ["Nirmala UI", "Tunga", "Noto Sans Kannada", "Kannada Sangam MN", "Lohit Kannada"], + [UnicodeScript.Malayalam] = ["Nirmala UI", "Kartika", "Noto Sans Malayalam", "Malayalam Sangam MN", "Lohit Malayalam"], + [UnicodeScript.Sinhala] = ["Nirmala UI", "Iskoola Pota", "Noto Sans Sinhala", "Sinhala Sangam MN"], + [UnicodeScript.Thai] = ["Leelawadee UI", "Tahoma", "Noto Sans Thai", "Thonburi", "Noto Serif Thai"], + [UnicodeScript.Lao] = ["Leelawadee UI", "Lao UI", "Noto Sans Lao", "Lao Sangam MN"], + [UnicodeScript.Khmer] = ["Leelawadee UI", "Khmer UI", "Noto Sans Khmer", "Khmer Sangam MN"], + [UnicodeScript.Myanmar] = ["Myanmar Text", "Noto Sans Myanmar", "Myanmar Sangam MN", "Padauk"], + [UnicodeScript.Tibetan] = ["Microsoft Himalaya", "Noto Serif Tibetan", "Kailasa"], + [UnicodeScript.Mongolian] = ["Mongolian Baiti", "Noto Sans Mongolian"], + [UnicodeScript.Ethiopic] = ["Ebrima", "Nyala", "Noto Sans Ethiopic", "Kefa"], + [UnicodeScript.Armenian] = ["Segoe UI", "Sylfaen", "Noto Sans Armenian", "DejaVu Sans"], + [UnicodeScript.Georgian] = ["Segoe UI", "Sylfaen", "Noto Sans Georgian", "DejaVu Sans"], + [UnicodeScript.Cherokee] = ["Gadugi", "Plantagenet Cherokee", "Noto Sans Cherokee"], + }; + + private readonly IFontCollection _fonts; + private readonly ConcurrentDictionary<(string Family, FontStyle Style), ResolvedFont?> _found = new(); + + /// The fonts to choose from; the installed ones when null. + public FontResolver(IFontCollection? fonts = null) => _fonts = fonts ?? SystemFonts.Instance; + + // ───────────────────────── Names ───────────────────────── + + /// + /// The installed face for a family name and style, by the name, the name without style words, and its other + /// names (); null when none of them is installed. + /// + public ResolvedFont? Find(string family, FontStyle style) => _found.GetOrAdd((family.Trim(), style), static (key, fonts) => + { + foreach (var (name, weight) in FontSubstitution.Names(key.Family)) + { + // A style word in the name sets the weight unless the run asks for another ("Pretendard Medium" bold). + var wanted = weight is { } named && key.Style.Weight == FontStyle.Regular.Weight ? key.Style with { Weight = named } : key.Style; + foreach (var candidate in FontNames.Alternatives(name).Prepend(name)) + { + var faces = fonts.Family(candidate); + if (faces.Count > 0) + return Match(faces, wanted); + } + } + return null; + }, _fonts); + + /// + /// The font to draw a family with: , else (or instead of a CFF2 font) an installed font for the + /// name's script, or the text's , in the same style (serif or sans). Generic families + /// (serif, sans-serif, monospace) and no family at all get the language's default fonts. Null only when no + /// candidate font is installed. + /// + /// "ko", "ja", "zh-Hans", "zh-Hant", or "" / null for other text. + public ResolvedFont? Resolve(string? family, FontStyle style, string? language = null) + { + language ??= ""; + if (string.IsNullOrWhiteSpace(family) || FontNames.IsGeneric(family)) + { + var generic = family?.Trim().ToLowerInvariant() ?? ""; + var names = generic.EndsWith("monospace", StringComparison.Ordinal) + ? FontNames.Monospace.Concat(FontSubstitution.Preferred(language, false)) + : FontSubstitution.Preferred(language, generic is "serif" or "ui-serif"); + return First(names, style) ?? First(FontSubstitution.Preferred("", false), style); + } + var found = Find(family, style); + if (found is { Face.Cff2: false }) + return found; + var script = FontSubstitution.Script(family) is { Length: > 0 } own ? own : language; + return First(FontSubstitution.Preferred(script, FontSubstitution.IsSerif(family)), style) + ?? found + ?? First(FontSubstitution.Preferred("", FontSubstitution.IsSerif(family)), style); + } + + /// + /// Document-wide replacements by (slot, face): Korean default fonts in Chinese or Japanese text, fonts that lack + /// the text's characters, CFF2 fonts (see ). + /// + /// The fonts the document names, with the slot each is for ("HANGUL", "LATIN", ...). + public Dictionary<(string Slot, string Face), string> Plan(IReadOnlyList<(string Slot, string Face)> slots, string text) => + FontSubstitution.Plan(slots, text, name => Find(name, FontStyle.Regular) is { } font + ? new FontCoverage(!font.Face.Cff2, font.Face.Has) + : null); + + /// The first of that is installed and not CFF2. + private ResolvedFont? First(IEnumerable names, FontStyle style) + { + foreach (var name in names) + { + if (Find(name, style) is { Face.Cff2: false } font) + return font; + } + return null; + } + + /// + /// The face of nearest : the slant first, then the weight by the + /// CSS rules (for 400–500 the weights up to 500, then lighter, then heavier; lighter first below 400, heavier + /// first above 500). + /// + internal static ResolvedFont Match(IReadOnlyList faces, FontStyle style) + { + var slanted = faces.Where(f => f.Italic == style.Italic).ToList(); + var pool = slanted.Count > 0 ? slanted : faces; + var wanted = style.Weight; + var best = pool.MinBy(f => WeightDistance(f.Weight, wanted))!; + return new ResolvedFont(best, SyntheticBold: style.Bold && best.Weight < 600, SyntheticItalic: style.Italic && !best.Italic); + } + + /// Order key of a face weight for a wanted weight (lower is better). + private static int WeightDistance(int weight, int wanted) + { + if (weight == wanted) + return 0; + if (wanted is >= 400 and <= 500) + { + if (weight > wanted && weight <= 500) + return weight - wanted; + if (weight < wanted) + return 1000 + wanted - weight; + return 2000 + weight - wanted; + } + if (wanted < 400) + return weight < wanted ? wanted - weight : 1000 + weight - wanted; + return weight > wanted ? weight - wanted : 1000 + wanted - weight; + } + + // ───────────────────────── Fallback ───────────────────────── + + /// + /// Splits text[start..start+length] into runs that each draw with one font: + /// where it has every character of a grapheme cluster, else the font found for the cluster. Clusters no installed + /// font has stay with the primary font (they draw as its missing-glyph box). + /// + /// The whole paragraph, so the language can be told from it. + /// "ko", "ja", "zh-Hans", "zh-Hant" or ""; null: told from . + public List Fallback(string text, int start, int length, ResolvedFont primary, FontStyle style, string? language = null) + { + ArgumentOutOfRangeException.ThrowIfNegative(start); + ArgumentOutOfRangeException.ThrowIfNegative(length); + ArgumentOutOfRangeException.ThrowIfGreaterThan(start + length, text.Length); + language ??= FontSubstitution.Language(text); + var serif = FontSubstitution.IsSerif(primary.Family); + var used = new List { primary }; + var runs = new List(); + var needed = new List(); + var end = start + length; + var position = start; + ResolvedFont? previous = null; + while (position < end) + { + var cluster = Math.Max(1, StringInfo.GetNextTextElementLength(text.AsSpan(position, end - position))); + Needed(text.AsSpan(position, cluster), needed); + var font = Choose(needed, primary, previous, used, style, language, serif); + if (!used.Contains(font)) + used.Add(font); + if (runs.Count > 0 && runs[^1].Font == font) + runs[^1] = runs[^1] with { Length = runs[^1].Length + cluster }; + else + runs.Add(new FontRun(position, cluster, font)); + previous = font; + position += cluster; + } + return runs; + } + + /// The font for one cluster. + private ResolvedFont Choose(List needed, ResolvedFont primary, ResolvedFont? previous, List used, + FontStyle style, string language, bool serif) + { + if (needed.Count == 0) + return previous ?? primary; + // Punctuation, digits and spaces keep the font of the text around them. + var neutral = needed.All(c => UnicodeData.GetScript(c) is UnicodeScript.Common or UnicodeScript.Inherited); + if (neutral && previous is not null && Covers(previous, needed)) + return previous; + if (Covers(primary, needed)) + return primary; + foreach (var font in used) + { + if (Covers(font, needed)) + return font; + } + + var script = needed.Select(UnicodeData.GetScript).FirstOrDefault(s => s is not (UnicodeScript.Common or UnicodeScript.Inherited)); + var tag = LanguageFor(script, language); + ResolvedFont? partial = null; + foreach (var name in Candidates(script, tag, serif)) + { + if (Find(name, style) is not { Face.Cff2: false } font) + continue; + if (Covers(font, needed)) + return font; + if (partial is null && font.Face.Has(needed[0])) + partial = font; + } + if (_fonts.ForCharacter(needed[0], style, tag.Length > 0 ? tag : null) is { } system) + { + var font = Match([system], style); + if (Covers(font, needed) || partial is null) + return font; + } + return partial ?? primary; + } + + /// Installed fonts to try for a script, best first. + private static IEnumerable Candidates(UnicodeScript script, string language, bool serif) + { + var own = ScriptFonts.TryGetValue(script, out var fonts) ? fonts : []; + return own.Concat(FontSubstitution.Preferred(language, serif)).Concat(FontSubstitution.Wide) + .Distinct(StringComparer.OrdinalIgnoreCase); + } + + /// The language that picks a script's fonts: Hangul is Korean, kana Japanese, Han the text's. + private static string LanguageFor(UnicodeScript script, string language) => script switch + { + UnicodeScript.Hangul => "ko", + UnicodeScript.Hiragana or UnicodeScript.Katakana or UnicodeScript.KatakanaOrHiragana => "ja", + UnicodeScript.Bopomofo => language is "zh-Hans" ? "zh-Hans" : "zh-Hant", + UnicodeScript.Han => language is "ko" or "ja" or "zh-Hant" ? language : "zh-Hans", + _ => language, + }; + + private static bool Covers(ResolvedFont font, List needed) + { + foreach (var codePoint in needed) + { + if (!font.Face.Has(codePoint)) + return false; + } + return true; + } + + /// + /// The code points of a cluster a font must have: all but the default-ignorable ones (controls, joiners, + /// direction marks, variation selectors), which the shaper hides. + /// + private static void Needed(ReadOnlySpan cluster, List needed) + { + needed.Clear(); + for (var i = 0; i < cluster.Length; i++) + { + var codePoint = UnicodeData.CodePointAt(cluster, i); + if (codePoint > 0xFFFF) + i++; + if (codePoint is >= 0xFE00 and <= 0xFE0F or >= 0xE0100 and <= 0xE01EF) + continue; + if (CharUnicodeInfo.GetUnicodeCategory(codePoint) is UnicodeCategory.Control or UnicodeCategory.Format + or UnicodeCategory.LineSeparator or UnicodeCategory.ParagraphSeparator) + continue; + needed.Add(codePoint); + } + } +} diff --git a/src/Filee.Engines/Typesetting/FontSlots.cs b/src/Filee.Engines/Typesetting/FontSlots.cs new file mode 100644 index 00000000..1986321e --- /dev/null +++ b/src/Filee.Engines/Typesetting/FontSlots.cs @@ -0,0 +1,179 @@ +// Which of a run's fonts draws a character. 한글 gives each character format seven language slots (HLanguageSlots: +// 한글, 영문, 한자, 일어, 외국어, 기호, 사용자) with a font, 장평, 자간, relative size and offset each; Word gives a run +// four fonts (HWordFonts: ascii, hAnsi, eastAsia, cs). The layout splits text into runs by slot before it resolves +// fonts and shapes. +// +// A grapheme cluster takes the slot of its first character, so combining marks, variation selectors and joiners stay +// with their base (one font per cluster; the shaper needs them together). +// +// 한글: by script (Hangul, Han, kana, Latin), with the Common characters of the ASCII / Latin-1 / spacing modifier range +// in 영문 like 한글 does, other punctuation and symbols (quotes, dashes, CJK punctuation, fullwidth forms, arrows, +// box drawing, emoji) in 기호, private use in 사용자, and every other script (Greek, Cyrillic, Arabic, Thai, ...) in +// 외국어. Word: ASCII in ascii, complex scripts (right-to-left and Brahmic / Southeast Asian scripts) in cs, CJK +// scripts and blocks in eastAsia and the rest in hAnsi. With w:hint="eastAsia", ambiguous-width characters (curly +// quotes, dashes, ellipsis, Greek and Cyrillic) and private use go to eastAsia, as Word does for East Asian text; +// Word's further distinction by the run's East Asian language (Latin-1 letters in Chinese text) is not made. + +using System.Globalization; +using Filee.Engines.Hwp.Hwpx; + +namespace Filee.Engines.Typesetting; + +/// Word's run fonts (w:rFonts). +internal enum WordFontSlot +{ + /// w:ascii: U+0000–U+007F. + Ascii, + + /// w:hAnsi: other Latin, Greek, Cyrillic, symbols, ... + HighAnsi, + + /// w:eastAsia: CJK text. + EastAsia, + + /// w:cs: complex-script text (Arabic, Hebrew, Indic, Thai, ...). + ComplexScript, +} + +/// Word's w:hint: which font ambiguous characters take. +internal enum WordFontHint +{ + Default, + EastAsia, + ComplexScript, +} + +/// A piece of text whose characters all fall into one font slot. +/// UTF-16 index of the first character. +/// Length in UTF-16 code units. +internal readonly record struct SlotRun(int Start, int Length, TSlot Slot) +{ + public int End => Start + Length; +} + +/// Character → font slot for 한글's language slots and Word's run fonts. +internal static class FontSlots +{ + /// The 한글 language slot of a character. + public static HLanguage Hwp(int codePoint) + { + if (IsPrivateUse(codePoint)) + return HLanguage.User; + switch (UnicodeData.GetScript(codePoint)) + { + case UnicodeScript.Hangul: + return HLanguage.Hangul; + case UnicodeScript.Han: + return HLanguage.Hanja; + case UnicodeScript.Hiragana or UnicodeScript.Katakana or UnicodeScript.KatakanaOrHiragana: + return HLanguage.Japanese; + case UnicodeScript.Latin: + return IsFullwidthForm(codePoint) ? HLanguage.Symbol : HLanguage.Latin; + case UnicodeScript.Common or UnicodeScript.Inherited: + if (codePoint < 0x0300) + return HLanguage.Latin; + // The prolonged sound mark and the kana punctuation (ー ・ ゛) are Common but Japanese. + if (codePoint is >= 0x3099 and <= 0x30FF or >= 0xFF70 and <= 0xFF9F) + return HLanguage.Japanese; + return HLanguage.Symbol; + default: + return HLanguage.Other; + } + } + + /// The Word run font a character is drawn with. + /// The run is complex-script text (w:cs or w:rtl): every character takes cs. + public static WordFontSlot Word(int codePoint, WordFontHint hint = WordFontHint.Default, bool complexScript = false) + { + if (complexScript) + return WordFontSlot.ComplexScript; + if (codePoint < 0x80) + return WordFontSlot.Ascii; + if (IsPrivateUse(codePoint)) + return hint == WordFontHint.EastAsia ? WordFontSlot.EastAsia : WordFontSlot.HighAnsi; + var script = UnicodeData.GetScript(codePoint); + if (IsComplexScript(script)) + return WordFontSlot.ComplexScript; + if (IsEastAsian(script, codePoint)) + return WordFontSlot.EastAsia; + if (hint == WordFontHint.EastAsia && UnicodeData.GetEastAsianWidth(codePoint) == EastAsianWidth.Ambiguous) + return WordFontSlot.EastAsia; + return WordFontSlot.HighAnsi; + } + + /// The font a Word run gives a slot (null when the run leaves it open). + public static string? Font(this HWordFonts fonts, WordFontSlot slot) => slot switch + { + WordFontSlot.Ascii => fonts.Ascii, + WordFontSlot.HighAnsi => fonts.HighAnsi, + WordFontSlot.EastAsia => fonts.EastAsia, + _ => fonts.ComplexScript, + }; + + /// Word's w:hint value as a . + public static WordFontHint Hint(string? hint) => hint switch + { + "eastAsia" => WordFontHint.EastAsia, + "cs" => WordFontHint.ComplexScript, + _ => WordFontHint.Default, + }; + + /// Splits text[start..start+length] into runs of one 한글 language slot. + public static List> HwpRuns(string text, int start, int length) => Runs(text, start, length, Hwp); + + /// Splits text[start..start+length] into runs of one Word run font. + public static List> WordRuns(string text, int start, int length, WordFontHint hint = WordFontHint.Default, + bool complexScript = false) => + Runs(text, start, length, c => Word(c, hint, complexScript)); + + /// + /// Splits text into runs of one slot: each grapheme cluster takes the slot of its first character, and neighbouring + /// clusters of the same slot form one run. + /// + public static List> Runs(string text, int start, int length, Func classify) + { + ArgumentOutOfRangeException.ThrowIfNegative(start); + ArgumentOutOfRangeException.ThrowIfNegative(length); + ArgumentOutOfRangeException.ThrowIfGreaterThan(start + length, text.Length); + var runs = new List>(); + var comparer = EqualityComparer.Default; + var end = start + length; + var position = start; + while (position < end) + { + var cluster = Math.Max(1, StringInfo.GetNextTextElementLength(text.AsSpan(position, end - position))); + var slot = classify(UnicodeData.CodePointAt(text, position)); + if (runs.Count > 0 && comparer.Equals(runs[^1].Slot, slot)) + runs[^1] = runs[^1] with { Length = runs[^1].Length + cluster }; + else + runs.Add(new SlotRun(position, cluster, slot)); + position += cluster; + } + return runs; + } + + private static bool IsPrivateUse(int codePoint) => + codePoint is >= 0xE000 and <= 0xF8FF or >= 0xF0000 and <= 0x10FFFD; + + /// The fullwidth ASCII variants (Ａ, ０, ！): CJK typographic forms, drawn by the East Asian font. + private static bool IsFullwidthForm(int codePoint) => codePoint is >= 0xFF01 and <= 0xFF60; + + /// Scripts Word draws with the cs font: right-to-left, Brahmic and Southeast Asian scripts. + private static bool IsComplexScript(UnicodeScript script) => script is UnicodeScript.Arabic or UnicodeScript.Hebrew + or UnicodeScript.Syriac or UnicodeScript.Thaana or UnicodeScript.Nko or UnicodeScript.Samaritan or UnicodeScript.Mandaic + or UnicodeScript.Devanagari or UnicodeScript.Bengali or UnicodeScript.Gurmukhi or UnicodeScript.Gujarati + or UnicodeScript.Oriya or UnicodeScript.Tamil or UnicodeScript.Telugu or UnicodeScript.Kannada or UnicodeScript.Malayalam + or UnicodeScript.Sinhala or UnicodeScript.Thai or UnicodeScript.Lao or UnicodeScript.Tibetan or UnicodeScript.Myanmar + or UnicodeScript.Khmer; + + /// CJK scripts and the CJK punctuation, symbol and compatibility blocks. + private static bool IsEastAsian(UnicodeScript script, int codePoint) => + script is UnicodeScript.Han or UnicodeScript.Hangul or UnicodeScript.Hiragana or UnicodeScript.Katakana + or UnicodeScript.KatakanaOrHiragana or UnicodeScript.Bopomofo or UnicodeScript.Yi + || codePoint is >= 0x2E80 and <= 0x2FFF // CJK radicals, Kangxi radicals, ideographic description + or >= 0x3000 and <= 0x303F // CJK symbols and punctuation + or >= 0x3190 and <= 0x33FF // kanbun, CJK strokes, enclosed CJK, CJK compatibility + or >= 0xFE10 and <= 0xFE1F // vertical forms + or >= 0xFE30 and <= 0xFE4F // CJK compatibility forms + or >= 0xFF00 and <= 0xFFEF; // halfwidth and fullwidth forms +} diff --git a/src/Filee.Engines/Typesetting/FontSubstitution.cs b/src/Filee.Engines/Typesetting/FontSubstitution.cs new file mode 100644 index 00000000..c3024277 --- /dev/null +++ b/src/Filee.Engines/Typesetting/FontSubstitution.cs @@ -0,0 +1,252 @@ +// Document-wide font replacements, shared by the layout engine's FontResolver and by RhwpFonts (which writes them into +// the HWPX copy rhwp draws). The rules came from RhwpFonts: +// +// - Language: documents Filee writes (Markdown, text, HTML → HWPX) name Korean fonts in every slot. In a Chinese or +// Japanese document those draw Korean-style Hanja and, for characters they lack, a different fallback font per +// character (issue #64). Without Hangul in the text, Korean fonts are swapped for an installed Chinese (Simplified +// or Traditional) or Japanese font of the same style. +// - Coverage: a slot's font draws a character when it has it, else a fallback does (Arabic came out in a calligraphic +// font). In text without Hangul, Han or kana, a slot's font is swapped for an installed one of the same style that +// has every character of the text the old one had, and more of the rest: Russian, Greek, Thai or Arabic text gets +// one consistent font. +// - Unusable fonts: a font the consumer can't use (rhwp: CFF2 variable fonts, issue #61, whose pages came out blank) +// is swapped for one of its script and style when one is installed. + +using System.Globalization; +using System.Text; + +namespace Filee.Engines.Typesetting; + +/// What the substitution rules need to know about an installed font. +/// False when the consumer can't use the font (rhwp: CFF2 outlines) and wants another. +/// Whether the font has a glyph for a code point. +internal record FontCoverage(bool Usable, Func Has); + +/// Language and coverage rules for replacing fonts, and what a font's name says about it. +internal static class FontSubstitution +{ + /// Style words a family name may end with ("思源宋体 VF Medium"), with the weight they stand for. + internal static readonly IReadOnlyDictionary StyleWords = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["Thin"] = 100, + ["ExtraLight"] = 200, + ["Light"] = 300, + ["Normal"] = 400, + ["Regular"] = 400, + ["Medium"] = 500, + ["SemiBold"] = 600, + ["Bold"] = 700, + ["ExtraBold"] = 800, + ["Heavy"] = 900, + ["Black"] = 900, + }; + + /// Installed fonts in order of preference, by script, serif first then sans. + private static readonly Dictionary Candidates = new() + { + ["zh-Hans"] = (["SimSun", "NSimSun", "Noto Serif SC", "Songti SC", "Noto Serif CJK SC"], + ["Microsoft YaHei", "DengXian", "Noto Sans SC", "PingFang SC", "Noto Sans CJK SC", "SimHei"]), + ["zh-Hant"] = (["PMingLiU", "MingLiU", "Noto Serif TC", "Songti TC", "Noto Serif CJK TC"], + ["Microsoft JhengHei", "Noto Sans TC", "PingFang TC", "Noto Sans CJK TC"]), + ["ja"] = (["Yu Mincho", "MS Mincho", "Noto Serif JP", "Hiragino Mincho ProN", "Noto Serif CJK JP"], + ["Yu Gothic", "Meiryo", "MS Gothic", "Noto Sans JP", "Hiragino Sans", "Noto Sans CJK JP"]), + ["ko"] = (["Batang", "HCR Batang", "Noto Serif KR", "AppleMyungjo", "Noto Serif CJK KR"], + ["Malgun Gothic", "Noto Sans KR", "Apple SD Gothic Neo", "NanumGothic", "Noto Sans CJK KR"]), + // Wide coverage first (Tahoma and Times New Roman also have Greek, Cyrillic, Arabic, Hebrew; Tahoma Thai). + [""] = (["Times New Roman", "Noto Serif", "DejaVu Serif", "Georgia", "Liberation Serif"], + ["Segoe UI", "Tahoma", "Arial", "Noto Sans", "DejaVu Sans", "Helvetica", "Liberation Sans"]), + }; + + /// Fonts for scripts a renderer's own fallback draws, which may have better ones installed. + private static readonly string[] WideFonts = ["Leelawadee UI", "Nirmala UI", "Microsoft Sans Serif", "Arial Unicode MS"]; + + /// + /// Simplified and Traditional forms of common characters: which form a Chinese text uses. Forms Japanese writes + /// the same way (国, 学, 会, 点, …) are left out. + /// + private const string SimplifiedOnly = "这们说为时个对发经过还进动东车长门间问见现实从开关书无让应该话样"; + private const string TraditionalOnly = "這們說為時個對發經過還進動東車長門間問見現實從開關書無讓應該話樣"; + + /// Replacements by (slot, face): the language, coverage and unusable-font rules above. + /// The fonts a document names, with the slot (language) each is used for. + /// The document's text. + /// An installed font by name, or null when it is not installed. + public static Dictionary<(string Slot, string Face), string> Plan( + IReadOnlyList<(string Slot, string Face)> slots, string text, Func lookup) + { + var characters = Characters(text); + var language = Language(text); + var result = new Dictionary<(string, string), string>(); + foreach (var (slot, face) in slots.Distinct()) + { + if (face.Length == 0) + continue; + var serif = IsSerif(face); + var chosen = face; + var current = lookup(face); + + // Korean fonts in a Chinese or Japanese text; a font that can't be used. + if (language is not ("ko" or "") && Script(face) == "ko" || current is { Usable: false }) + { + var script = current is { Usable: false } && Script(face) is { Length: > 0 } own ? own : language; + if (Preferred(script, serif).FirstOrDefault(name => lookup(name) is { Usable: true }) is { } replacement) + chosen = replacement; + } + + // Text without Hangul, Han or kana (Russian, Greek, Thai, Arabic, …): a font of the same style with every + // character the chosen one has, and more of the others. In Korean, Chinese and Japanese text the language + // font stays, and the fallback draws the odd foreign word as before. + var mine = language != "" || slot is "HANJA" or "JAPANESE" + ? [] + : characters.Where(c => !IsHan(c) && !IsKana(c) && (slot == "HANGUL" || !IsHangul(c))).ToList(); + if (mine.Count > 0 && lookup(chosen) is { Usable: true } font) + { + var covered = mine.Where(c => font.Has(c)).ToList(); + if (covered.Count < mine.Count) + { + var best = chosen; + var bestCount = covered.Count; + var sameStyle = (serif ? Styled(language).Serif.Concat(Styled("").Serif) : Styled(language).Sans.Concat(Styled("").Sans).Concat(WideFonts)) + .Distinct(StringComparer.OrdinalIgnoreCase); + foreach (var name in sameStyle) + { + if (lookup(name) is not { Usable: true } other || !covered.All(other.Has)) + continue; + var count = mine.Count(other.Has); + if (count == mine.Count) + { + (best, bestCount) = (name, count); // the first one in order that has everything + break; + } + if (count > bestCount) + (best, bestCount) = (name, count); + } + chosen = best; + } + } + + if (!string.Equals(chosen, face, StringComparison.Ordinal)) + result[(slot, face)] = chosen; + } + return result; + } + + /// + /// Installed-font candidates for a language ("ko", "ja", "zh-Hans", "zh-Hant", "" for other text) in order of + /// preference: the requested style (serif or sans) first, then the other, then wide-coverage fonts. + /// + public static IEnumerable Preferred(string script, bool serif) + { + var (serifs, sans) = Styled(script); + return serif ? serifs.Concat(sans) : sans.Concat(serifs); + } + + /// Fonts with wide script coverage (Thai, Indic, ...) to try after the language's own. + public static IReadOnlyList Wide => WideFonts; + + private static (string[] Serif, string[] Sans) Styled(string script) => + Candidates.TryGetValue(script, out var c) ? c : Candidates[""]; + + private static bool IsHangul(int c) => c is >= 0xAC00 and <= 0xD7A3 or >= 0x1100 and <= 0x11FF or >= 0x3130 and <= 0x318F; + + private static bool IsKana(int c) => c is >= 0x3040 and <= 0x30FF; + + private static bool IsHan(int c) => c is >= 0x4E00 and <= 0x9FFF or >= 0x3400 and <= 0x4DBF or >= 0x20000 and <= 0x2FFFF; + + /// + /// The language the Han characters of a text are in: Korean with Hangul, Japanese with kana, else Simplified or + /// Traditional Chinese by common characters; "" without Han characters. + /// + public static string Language(string text) + { + int hangul = 0, kana = 0, han = 0, simplified = 0, traditional = 0; + foreach (var c in text) + { + if (c is >= '가' and <= '힣' or >= 'ᄀ' and <= 'ᇿ' or >= '㄰' and <= '㆏') + hangul++; + else if (c is >= '぀' and <= 'ヿ') + kana++; + else if (c is >= '一' and <= '鿿' or >= '㐀' and <= '䶿') + { + han++; + if (SimplifiedOnly.Contains(c)) + simplified++; + else if (TraditionalOnly.Contains(c)) + traditional++; + } + } + if (hangul > 0) + return "ko"; + // Japanese prose is mostly kana; a Chinese text quoting a Japanese title is not. + if (kana > 0 && (simplified + traditional == 0 || kana * 3 >= han)) + return "ja"; + if (han > 0) + return traditional > simplified ? "zh-Hant" : "zh-Hans"; + return ""; + } + + /// Distinct characters of the text that need a glyph (no spaces or controls), at most 4000. + private static List Characters(string text) + { + var set = new HashSet(); + foreach (var rune in text.EnumerateRunes()) + { + if (set.Count >= 4000) + break; + if (NeedsGlyph(rune.Value)) + set.Add(rune.Value); + } + return set.ToList(); + } + + /// + /// False for characters no font needs to have: spaces, controls, format characters (joiners, direction marks), + /// combining marks drawn with their base, private use and unassigned code points. + /// + public static bool NeedsGlyph(int codePoint) => + Rune.IsValid(codePoint) && Rune.GetUnicodeCategory(new Rune(codePoint)) is not (UnicodeCategory.SpaceSeparator + or UnicodeCategory.Control or UnicodeCategory.Format or UnicodeCategory.LineSeparator + or UnicodeCategory.ParagraphSeparator or UnicodeCategory.NonSpacingMark or UnicodeCategory.PrivateUse + or UnicodeCategory.OtherNotAssigned); + + /// + /// The names to look a font up by: the name itself, then without trailing style words ("思源宋体 VF Medium" names a + /// style of "思源宋体 VF"), each with the weight the dropped word stands for. + /// + public static IEnumerable<(string Name, int? Weight)> Names(string face) + { + yield return (face, null); + var words = face.Split(' ', StringSplitOptions.RemoveEmptyEntries).ToList(); + int? weight = null; + while (words.Count > 1 && StyleWords.TryGetValue(words[^1], out var stands)) + { + weight ??= stands; + words.RemoveAt(words.Count - 1); + yield return (string.Join(' ', words), weight); + } + } + + /// The script a font name points to: "思源宋体" and "SC" are Simplified Chinese, "명조" Korean, …. + public static string Script(string face) + { + bool Has(params string[] parts) => parts.Any(p => face.Contains(p, StringComparison.OrdinalIgnoreCase)); + bool Word(params string[] words) => face.Split(' ', '-', '_').Any(w => words.Contains(w, StringComparer.OrdinalIgnoreCase)); + if (Word("SC", "CN", "GB") || Has("简", "宋体", "黑体", "仿宋", "楷体")) + return "zh-Hans"; + if (Word("TC", "HK", "TW") || Has("繁", "明體", "黑體", "宋體")) + return "zh-Hant"; + if (Word("JP", "J") || Has("明朝", "ゴシック", "游")) + return "ja"; + if (Word("KR", "K") || face.Any(c => c is >= '가' and <= '힣') + || Candidates["ko"].Serif.Concat(Candidates["ko"].Sans).Contains(face, StringComparer.OrdinalIgnoreCase)) + return "ko"; + if (face.Any(c => c is >= '一' and <= '鿿')) + return "zh-Hans"; + return ""; + } + + /// Serif (Song/Ming/Mincho/Batang) rather than sans (Hei/Gothic/Dotum). + public static bool IsSerif(string face) => + new[] { "Serif", "Song", "Ming", "Mincho", "Myeongjo", "Myungjo", "Batang", "Times", "宋", "明", "바탕", "명조", "仿宋", "楷" } + .Any(p => face.Contains(p, StringComparison.OrdinalIgnoreCase)); +} diff --git a/src/Filee.Engines/Typesetting/Shaper.cs b/src/Filee.Engines/Typesetting/Shaper.cs new file mode 100644 index 00000000..9b424d01 --- /dev/null +++ b/src/Filee.Engines/Typesetting/Shaper.cs @@ -0,0 +1,340 @@ +// Text shaping for the layout engine: HarfBuzz turns a run of text in one font into glyph IDs with advances, offsets +// and the cluster each glyph belongs to, which is what the page model's GlyphRun takes (Rendering/PageContent.cs). +// Arabic joining, Indic conjuncts and reordered vowel signs, Thai marks and Hangul jamo composition all happen here. +// +// Generalised from Vector/GlyphOutlines.cs (which shapes SVG text for outlines): +// - Results are in font units (HarfBuzz scaled to the font's units per em), so one shaping serves every size; they are +// cached by font, features, script, direction, language and the run's text with its context. +// - A run is shaped inside its paragraph: up to five characters on either side go to HarfBuzz as context, so a word +// split into two runs (a colour or font change) still joins (Arabic) and reorders (Indic) as one. The context is part +// of the cache key. +// - HarfBuzz reads the font from PageFont.Data, pinned for as long as HarfBuzz's face lives: memory HarfBuzz doesn't +// own must outlive it (Blob.FromStream let HarfBuzz read freed memory in the PDF backend spike). +// - Clusters are HarfBuzz's default (monotone graphemes): marks join their base's cluster, and the clusters run in +// visual order (descending for right-to-left runs), as PageContent expects. +// +// Variable fonts shape at their default instance: HarfBuzzSharp has no binding for variation coordinates, and the PDF +// writer subsets the default instance too (see FontFace.Variable). + +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using Filee.Engines.Rendering; +using HarfBuzzSharp; +using SkiaSharp; +using Buffer = HarfBuzzSharp.Buffer; +using HbFont = HarfBuzzSharp.Font; +using HbScript = HarfBuzzSharp.Script; + +namespace Filee.Engines.Typesetting; + +internal enum TextDirection +{ + LeftToRight, + RightToLeft, +} + +/// An OpenType feature set for a whole run ("liga" off, "smcp" on, "salt" = 2). +internal readonly record struct FontFeature(string Tag, uint Value = 1) +{ + public override string ToString() => $"{Tag}={Value}"; +} + +/// How a run is shaped; null members are taken from the text. +/// The run's direction from the bidirectional algorithm; null: that of . +/// Null: the first script of the run other than Common and Inherited. +/// BCP 47 language ("ko", "sr", "zh-Hant"): selects language forms (OpenType locl). +internal sealed record ShapeOptions( + TextDirection? Direction = null, + UnicodeScript? Script = null, + string? Language = null, + IReadOnlyList? Features = null) +{ + public static readonly ShapeOptions Default = new(); +} + +/// +/// A shaped run in font units (multiply by size / ). The arrays hold one entry per glyph in +/// visual order. are UTF-16 offsets from the start of the run. The arrays are shared with the +/// shaper's cache and must not be changed. +/// +internal sealed class ShapedRun +{ + internal ShapedRun(int unitsPerEm, TextDirection direction, int length, ushort[] glyphs, int[] clusters, int[] xAdvances, + int[] yAdvances, int[] xOffsets, int[] yOffsets, bool[] unsafeToBreak) + { + UnitsPerEm = unitsPerEm; + Direction = direction; + Length = length; + Glyphs = glyphs; + Clusters = clusters; + XAdvances = xAdvances; + YAdvances = yAdvances; + XOffsets = xOffsets; + YOffsets = yOffsets; + UnsafeToBreak = unsafeToBreak; + Advance = xAdvances.Sum(); + } + + public int UnitsPerEm { get; } + + public TextDirection Direction { get; } + + /// Length of the shaped text in UTF-16 code units. + public int Length { get; } + + public ushort[] Glyphs { get; } + + /// Per glyph: the UTF-16 offset from the run start where its cluster starts. + public int[] Clusters { get; } + + public int[] XAdvances { get; } + + public int[] YAdvances { get; } + + /// Horizontal offset of each glyph from its pen position. + public int[] XOffsets { get; } + + /// Vertical offset of each glyph from its pen position, upwards (HarfBuzz's convention). + public int[] YOffsets { get; } + + /// + /// Per glyph: breaking the line before this glyph's cluster changes the shaping, so the text on either side must + /// be shaped again (HarfBuzz's UNSAFE_TO_BREAK). + /// + public bool[] UnsafeToBreak { get; } + + /// Total horizontal advance. + public int Advance { get; } + + /// The run's width at points (or any unit the size is in). + public float Width(float size) => Advance * size / UnitsPerEm; + + /// + /// The run as the page model draws it, the pen starting at (page coordinates, y down) + /// and moving right; is the run's text. + /// + /// Horizontal scale of glyphs and advances (한글 장평, Word w:w), 1 = as designed. + public GlyphRun ToGlyphRun(PageFont font, float size, SKPoint origin, string text, float scaleX = 1) + { + if (text.Length != Length) + throw new ArgumentException("The text is not the text of the run.", nameof(text)); + var scale = size / UnitsPerEm; + var positions = new SKPoint[Glyphs.Length]; + float x = origin.X, y = origin.Y; + for (var i = 0; i < Glyphs.Length; i++) + { + positions[i] = new SKPoint(x + XOffsets[i] * scale * scaleX, y - YOffsets[i] * scale); + x += XAdvances[i] * scale * scaleX; + y -= YAdvances[i] * scale; + } + return new GlyphRun(font, size, Glyphs, positions, text, Clusters) { ScaleX = scaleX }; + } +} + +/// Vertical and decoration metrics of a font in font units (y up: descenders are negative). +/// The typographic ascender HarfBuzz uses (OS/2 typo with USE_TYPO_METRICS, else hhea). +/// OS/2 usWinAscent (Word's line height on Windows comes from the Win metrics). +/// OS/2 usWinDescent, as a positive distance below the baseline. +internal sealed record FontMetrics( + int UnitsPerEm, int Ascender, int Descender, int LineGap, int WinAscent, int WinDescent, int XHeight, int CapHeight, + int UnderlinePosition, int UnderlineThickness, int StrikeoutPosition, int StrikeoutThickness, + int SuperscriptYOffset, int SuperscriptYSize, int SubscriptYOffset, int SubscriptYSize); + +/// Shapes runs with HarfBuzz and caches the results; one per layout job (not thread-safe). +internal sealed class Shaper : IDisposable +{ + /// Characters of context HarfBuzz looks at on either side of a run (HB_BUFFER_CONTEXT_LENGTH). + private const int ContextLength = 5; + + /// Shaped runs kept; the cache starts over when it is full. + private const int MaxCached = 20_000; + + private readonly Dictionary _fonts = new(ReferenceEqualityComparer.Instance); + private readonly Dictionary _cache = []; + + /// Runs shaped by HarfBuzz so far (cache misses), for tests. + internal int Shaped { get; private set; } + + /// Shapes all of . + public ShapedRun Shape(PageFont font, string text, ShapeOptions? options = null) => Shape(font, text, 0, text.Length, options); + + /// + /// Shapes text[start..start+length] with the characters around it as context; clusters are counted from + /// . + /// + public ShapedRun Shape(PageFont font, string text, int start, int length, ShapeOptions? options = null) + { + ArgumentOutOfRangeException.ThrowIfNegative(start); + ArgumentOutOfRangeException.ThrowIfNegative(length); + ArgumentOutOfRangeException.ThrowIfGreaterThan(start + length, text.Length); + options ??= ShapeOptions.Default; + var script = options.Script ?? ScriptOf(text, start, length); + var direction = options.Direction ?? DirectionOf(script); + var language = options.Language ?? ""; + var features = options.Features is { Count: > 0 } list ? string.Join(",", list) : ""; + + var contextStart = ContextStart(text, start); + var contextEnd = ContextEnd(text, start + length); + // The text's start and end (BOT / EOT) only when the run reaches them: else the context stands for the rest. + var key = new ShapeKey(font, text[contextStart..contextEnd], start - contextStart, length, start == 0, + start + length == text.Length, direction, script, language, features); + if (_cache.TryGetValue(key, out var cached)) + return cached; + + var run = Run(For(font), key, options.Features); + if (_cache.Count >= MaxCached) + _cache.Clear(); + _cache[key] = run; + Shaped++; + return run; + } + + /// The font's vertical and decoration metrics. + public FontMetrics Metrics(PageFont font) + { + var hb = For(font); + var metrics = hb.Font.OpenTypeMetrics; + int Get(OpenTypeMetricsTag tag) => metrics.TryGetPosition(tag, out var value) ? value : 0; + hb.Font.TryGetHorizontalFontExtents(out var extents); + return new FontMetrics(hb.UnitsPerEm, extents.Ascender, extents.Descender, extents.LineGap, + Get(OpenTypeMetricsTag.HorizontalClippingAscent), Get(OpenTypeMetricsTag.HorizontalClippingDescent), + Get(OpenTypeMetricsTag.XHeight), Get(OpenTypeMetricsTag.CapHeight), + Get(OpenTypeMetricsTag.UnderlineOffset), Get(OpenTypeMetricsTag.UnderlineSize), + Get(OpenTypeMetricsTag.StrikeoutOffset), Get(OpenTypeMetricsTag.StrikeoutSize), + Get(OpenTypeMetricsTag.SuperScriptEmYOffset), Get(OpenTypeMetricsTag.SuperScriptEmYSize), + Get(OpenTypeMetricsTag.SubScriptEmYOffset), Get(OpenTypeMetricsTag.SubScriptEmYSize)); + } + + private static ShapedRun Run(HarfBuzzFont hb, ShapeKey key, IReadOnlyList? features) + { + using var buffer = new Buffer(); + buffer.AddUtf16(key.Text, key.Start, key.Length); + buffer.Direction = key.Direction == TextDirection.RightToLeft ? Direction.RightToLeft : Direction.LeftToRight; + buffer.Script = HbScript.TryParse(key.Script.IsoCode(), out var script) ? script : HbScript.Common; + if (key.Language.Length > 0) + buffer.Language = new Language(key.Language); + var flags = BufferFlags.Default; + if (key.BeginningOfText) + flags |= BufferFlags.BeginningOfText; + if (key.EndOfText) + flags |= BufferFlags.EndOfText; + buffer.Flags = flags; + hb.Font.Shape(buffer, features is { Count: > 0 } ? [.. features.Select(f => new Feature(Tag.Parse(f.Tag), f.Value))] : []); + + var infos = buffer.GetGlyphInfoSpan(); + var positions = buffer.GetGlyphPositionSpan(); + var count = infos.Length; + var glyphs = new ushort[count]; + var clusters = new int[count]; + int[] xAdvances = new int[count], yAdvances = new int[count], xOffsets = new int[count], yOffsets = new int[count]; + var unsafeToBreak = new bool[count]; + for (var i = 0; i < count; i++) + { + glyphs[i] = (ushort)infos[i].Codepoint; + clusters[i] = (int)infos[i].Cluster - key.Start; + unsafeToBreak[i] = (infos[i].GlyphFlags & GlyphFlags.UnsafeToBreak) != 0; + xAdvances[i] = positions[i].XAdvance; + yAdvances[i] = positions[i].YAdvance; + xOffsets[i] = positions[i].XOffset; + yOffsets[i] = positions[i].YOffset; + } + return new ShapedRun(hb.UnitsPerEm, key.Direction, key.Length, glyphs, clusters, xAdvances, yAdvances, xOffsets, yOffsets, unsafeToBreak); + } + + /// The first script of the run other than Common and Inherited; Common when it has none. + internal static UnicodeScript ScriptOf(string text, int start, int length) + { + for (var i = start; i < start + length; i++) + { + var codePoint = UnicodeData.CodePointAt(text, i); + if (codePoint > 0xFFFF) + i++; + if (UnicodeData.GetScript(codePoint) is not (UnicodeScript.Common or UnicodeScript.Inherited) and var script) + return script; + } + return UnicodeScript.Common; + } + + /// The horizontal direction a script is written in. + internal static TextDirection DirectionOf(UnicodeScript script) => + HbScript.TryParse(script.IsoCode(), out var hb) && hb.HorizontalDirection == Direction.RightToLeft + ? TextDirection.RightToLeft + : TextDirection.LeftToRight; + + /// Where the context before begins: up to five code points. + private static int ContextStart(string text, int start) + { + var position = start; + for (var n = 0; n < ContextLength && position > 0; n++) + position -= position >= 2 && char.IsSurrogatePair(text[position - 2], text[position - 1]) ? 2 : 1; + return position; + } + + /// Where the context after ends: up to five code points. + private static int ContextEnd(string text, int end) + { + var position = end; + for (var n = 0; n < ContextLength && position < text.Length; n++) + position += position + 1 < text.Length && char.IsSurrogatePair(text[position], text[position + 1]) ? 2 : 1; + return position; + } + + private HarfBuzzFont For(PageFont font) + { + if (!_fonts.TryGetValue(font, out var hb)) + _fonts[font] = hb = new HarfBuzzFont(font); + return hb; + } + + public void Dispose() + { + foreach (var font in _fonts.Values) + font.Dispose(); + _fonts.Clear(); + _cache.Clear(); + } + + /// What a result needs: Text is the run with its context, the run is at Start. + private readonly record struct ShapeKey(PageFont Font, string Text, int Start, int Length, bool BeginningOfText, bool EndOfText, + TextDirection Direction, UnicodeScript Script, string Language, string Features) + { + public bool Equals(ShapeKey other) => ReferenceEquals(Font, other.Font) && Text == other.Text && Start == other.Start + && Length == other.Length && BeginningOfText == other.BeginningOfText && EndOfText == other.EndOfText + && Direction == other.Direction && Script == other.Script && Language == other.Language && Features == other.Features; + + public override int GetHashCode() => HashCode.Combine(RuntimeHelpers.GetHashCode(Font), Text, Start, Length, + Direction, Script, Language, Features); + } + + /// A HarfBuzz font over a PageFont's data, which stays pinned while HarfBuzz may read it. + private sealed class HarfBuzzFont : IDisposable + { + private readonly Blob _blob; + private readonly Face _face; + + public HarfBuzzFont(PageFont font) + { + var data = font.Data; + var pin = GCHandle.Alloc(data, GCHandleType.Pinned); + // HarfBuzz calls the release delegate when it destroys the blob, after the face and font that use it. + _blob = new Blob(pin.AddrOfPinnedObject(), data.Length, MemoryMode.ReadOnly, () => pin.Free()); + _blob.MakeImmutable(); + _face = new Face(_blob, font.FaceIndex); + UnitsPerEm = _face.UnitsPerEm; + Font = new HbFont(_face); + Font.SetFunctionsOpenType(); + Font.SetScale(UnitsPerEm, UnitsPerEm); + } + + public HbFont Font { get; } + + public int UnitsPerEm { get; } + + public void Dispose() + { + Font.Dispose(); + _face.Dispose(); + _blob.Dispose(); + } + } +} diff --git a/src/Filee.Engines/Typesetting/SystemFonts.cs b/src/Filee.Engines/Typesetting/SystemFonts.cs new file mode 100644 index 00000000..edfbe80c --- /dev/null +++ b/src/Filee.Engines/Typesetting/SystemFonts.cs @@ -0,0 +1,97 @@ +// The fonts installed on the computer, through Skia's font manager: DirectWrite on Windows, CoreText on macOS and +// fontconfig on Linux. Nothing is downloaded or shipped. +// +// - Windows matches localized family names ("맑은 고딕") and returns nothing for unknown ones. fontconfig (and +// CoreText) answer an unknown family with some installed font, so elsewhere only faces whose family name is the +// requested one count. +// - Style, variable and CFF2 flags come from the font's own tables (OS/2, fvar, CFF2), not from what the font +// manager reports: DirectWrite lists the named instances of a variable font as faces of their own, but their data +// is the whole font, which shapes and draws at its default instance. +// - Typefaces are never disposed: SkiaSharp hands out one wrapper per native typeface, so it may be the very object +// the UI, Svg.Skia or the CAD fonts still draw with (see RhwpFonts.Find). + +using System.Buffers.Binary; +using System.Collections.Concurrent; +using Filee.Engines.Rendering; +using SkiaSharp; + +namespace Filee.Engines.Typesetting; + +/// The installed fonts. +internal sealed class SystemFonts : IFontCollection +{ + private static readonly uint Os2 = Tag("OS/2"), Fvar = Tag("fvar"), Cff2 = Tag("CFF2"); + + private readonly ConcurrentDictionary> _families = new(StringComparer.OrdinalIgnoreCase); + private readonly ConcurrentDictionary<(string Family, int Weight, bool Italic), FontFace> _fallbacks = new(); + + private SystemFonts() + { + } + + public static SystemFonts Instance { get; } = new(); + + /// Forgets the fonts looked up so far (). + internal static void ReleaseCache() + { + Instance._families.Clear(); + Instance._fallbacks.Clear(); + } + + public IReadOnlyList Family(string name) => _families.GetOrAdd(name, static name => + { + using var styles = SKFontManager.Default.GetFontStyles(name); + if (styles is null) + return []; + var faces = new List(); + for (var i = 0; i < styles.Count; i++) + { + var typeface = styles.CreateTypeface(i); + if (typeface is null) + continue; + if (!OperatingSystem.IsWindows() && !string.Equals(typeface.FamilyName, name, StringComparison.OrdinalIgnoreCase)) + continue; + var face = Face(name, typeface); + // The named instances of a variable font are one face: the same data, the same default instance. + if (!faces.Any(f => f.Weight == face.Weight && f.Italic == face.Italic)) + faces.Add(face); + } + return faces; + }); + + public FontFace? ForCharacter(int codePoint, FontStyle style, string? language) + { + var typeface = SKFontManager.Default.MatchCharacter(null, style.Weight, (int)SKFontStyleWidth.Normal, + style.Italic ? SKFontStyleSlant.Italic : SKFontStyleSlant.Upright, language is null ? [] : [language], codePoint); + if (typeface is null || typeface.GetGlyph(codePoint) == 0) + return null; + var probe = Face(typeface.FamilyName, typeface); + return _fallbacks.GetOrAdd((typeface.FamilyName, probe.Weight, probe.Italic), _ => probe); + } + + /// A face over a typeface Skia found, with its style read from the font's tables. + private static FontFace Face(string family, SKTypeface typeface) + { + var tags = typeface.GetTableTags() ?? []; + var weight = typeface.FontWeight; + var italic = typeface.FontSlant != SKFontStyleSlant.Upright; + if (typeface.TryGetTableData(Os2, out var os2) && os2.Length >= 64) + { + // Some old fonts store the weight class as 1–9. + weight = BinaryPrimitives.ReadUInt16BigEndian(os2.AsSpan(4)) switch + { + 0 => weight, + < 10 and var small => small * 100, + var value => Math.Min((int)value, 1000), + }; + italic = (os2[63] & 1) != 0; // fsSelection bit 0: ITALIC + } + return new FontFace(family, weight, italic, codePoint => typeface.GetGlyph(codePoint) != 0, () => PageFont.FromTypeface(typeface)) + { + Variable = tags.Contains(Fvar), + Cff2 = tags.Contains(Cff2), + }; + } + + private static uint Tag(string tag) => (uint)tag[0] << 24 | (uint)tag[1] << 16 | (uint)tag[2] << 8 | tag[3]; +} diff --git a/src/Filee.Engines/Typesetting/UnicodeData.Generated.cs b/src/Filee.Engines/Typesetting/UnicodeData.Generated.cs new file mode 100644 index 00000000..af6dcf6c --- /dev/null +++ b/src/Filee.Engines/Typesetting/UnicodeData.Generated.cs @@ -0,0 +1,1847 @@ +// +// Generated by build/generate-unicode-tables.ps1 from the Unicode Character Database 16.0.0 (LineBreak.txt, +// DerivedBidiClass.txt, Scripts.txt, EastAsianWidth.txt and PropertyValueAliases.txt). Do not edit: run the +// script. The data is Copyright (c) Unicode, Inc., under the Unicode License v3 (see THIRD-PARTY-NOTICES.md). +// + +namespace Filee.Engines.Typesetting; + +/// Line_Break property values (UAX #14), by their short names. +internal enum LineBreakClass : byte +{ + /// Unknown + XX, + + /// Ambiguous + AI, + + /// Aksara + AK, + + /// Alphabetic + AL, + + /// Aksara Prebase + AP, + + /// Aksara Start + AS, + + /// Break Both + B2, + + /// Break After + BA, + + /// Break Before + BB, + + /// Mandatory Break + BK, + + /// Contingent Break + CB, + + /// Conditional Japanese Starter + CJ, + + /// Close Punctuation + CL, + + /// Combining Mark + CM, + + /// Close Parenthesis + CP, + + /// Carriage Return + CR, + + /// E Base + EB, + + /// E Modifier + EM, + + /// Exclamation + EX, + + /// Glue + GL, + + /// H2 + H2, + + /// H3 + H3, + + /// Hebrew Letter + HL, + + /// Hyphen + HY, + + /// Ideographic + ID, + + /// Inseparable + IN, + + /// Infix Numeric + IS, + + /// JL + JL, + + /// JT + JT, + + /// JV + JV, + + /// Line Feed + LF, + + /// Next Line + NL, + + /// Nonstarter + NS, + + /// Numeric + NU, + + /// Open Punctuation + OP, + + /// Postfix Numeric + PO, + + /// Prefix Numeric + PR, + + /// Quotation + QU, + + /// Regional Indicator + RI, + + /// Complex Context + SA, + + /// Surrogate + SG, + + /// Space + SP, + + /// Break Symbols + SY, + + /// Virama Final + VF, + + /// Virama + VI, + + /// Word Joiner + WJ, + + /// ZWSpace + ZW, + + /// ZWJ + ZWJ, +} + +/// Bidi_Class property values (UAX #9), by their short names. +internal enum BidiClass : byte +{ + /// Left To Right + L, + + /// Arabic Letter + AL, + + /// Arabic Number + AN, + + /// Paragraph Separator + B, + + /// Boundary Neutral + BN, + + /// Common Separator + CS, + + /// European Number + EN, + + /// European Separator + ES, + + /// European Terminator + ET, + + /// First Strong Isolate + FSI, + + /// Left To Right Embedding + LRE, + + /// Left To Right Isolate + LRI, + + /// Left To Right Override + LRO, + + /// Nonspacing Mark + NSM, + + /// Other Neutral + ON, + + /// Pop Directional Format + PDF, + + /// Pop Directional Isolate + PDI, + + /// Right To Left + R, + + /// Right To Left Embedding + RLE, + + /// Right To Left Isolate + RLI, + + /// Right To Left Override + RLO, + + /// Segment Separator + S, + + /// White Space + WS, +} + +/// East_Asian_Width property values (UAX #11). +internal enum EastAsianWidth : byte +{ + /// N + Neutral, + + /// A + Ambiguous, + + /// F + Fullwidth, + + /// H + Halfwidth, + + /// Na + Narrow, + + /// W + Wide, +} + +/// Script property values (UAX #24); the summary is the ISO 15924 code. +internal enum UnicodeScript : byte +{ + /// Zzzz + Unknown, + + /// Zyyy + Common, + + /// Zinh + Inherited, + + /// Adlm + Adlam, + + /// Aghb + CaucasianAlbanian, + + /// Ahom + Ahom, + + /// Arab + Arabic, + + /// Armi + ImperialAramaic, + + /// Armn + Armenian, + + /// Avst + Avestan, + + /// Bali + Balinese, + + /// Bamu + Bamum, + + /// Bass + BassaVah, + + /// Batk + Batak, + + /// Beng + Bengali, + + /// Bhks + Bhaiksuki, + + /// Bopo + Bopomofo, + + /// Brah + Brahmi, + + /// Brai + Braille, + + /// Bugi + Buginese, + + /// Buhd + Buhid, + + /// Cakm + Chakma, + + /// Cans + CanadianAboriginal, + + /// Cari + Carian, + + /// Cham + Cham, + + /// Cher + Cherokee, + + /// Chrs + Chorasmian, + + /// Copt + Coptic, + + /// Cpmn + CyproMinoan, + + /// Cprt + Cypriot, + + /// Cyrl + Cyrillic, + + /// Deva + Devanagari, + + /// Diak + DivesAkuru, + + /// Dogr + Dogra, + + /// Dsrt + Deseret, + + /// Dupl + Duployan, + + /// Egyp + EgyptianHieroglyphs, + + /// Elba + Elbasan, + + /// Elym + Elymaic, + + /// Ethi + Ethiopic, + + /// Gara + Garay, + + /// Geor + Georgian, + + /// Glag + Glagolitic, + + /// Gong + GunjalaGondi, + + /// Gonm + MasaramGondi, + + /// Goth + Gothic, + + /// Gran + Grantha, + + /// Grek + Greek, + + /// Gujr + Gujarati, + + /// Gukh + GurungKhema, + + /// Guru + Gurmukhi, + + /// Hang + Hangul, + + /// Hani + Han, + + /// Hano + Hanunoo, + + /// Hatr + Hatran, + + /// Hebr + Hebrew, + + /// Hira + Hiragana, + + /// Hluw + AnatolianHieroglyphs, + + /// Hmng + PahawhHmong, + + /// Hmnp + NyiakengPuachueHmong, + + /// Hrkt + KatakanaOrHiragana, + + /// Hung + OldHungarian, + + /// Ital + OldItalic, + + /// Java + Javanese, + + /// Kali + KayahLi, + + /// Kana + Katakana, + + /// Kawi + Kawi, + + /// Khar + Kharoshthi, + + /// Khmr + Khmer, + + /// Khoj + Khojki, + + /// Kits + KhitanSmallScript, + + /// Knda + Kannada, + + /// Krai + KiratRai, + + /// Kthi + Kaithi, + + /// Lana + TaiTham, + + /// Laoo + Lao, + + /// Latn + Latin, + + /// Lepc + Lepcha, + + /// Limb + Limbu, + + /// Lina + LinearA, + + /// Linb + LinearB, + + /// Lisu + Lisu, + + /// Lyci + Lycian, + + /// Lydi + Lydian, + + /// Mahj + Mahajani, + + /// Maka + Makasar, + + /// Mand + Mandaic, + + /// Mani + Manichaean, + + /// Marc + Marchen, + + /// Medf + Medefaidrin, + + /// Mend + MendeKikakui, + + /// Merc + MeroiticCursive, + + /// Mero + MeroiticHieroglyphs, + + /// Mlym + Malayalam, + + /// Modi + Modi, + + /// Mong + Mongolian, + + /// Mroo + Mro, + + /// Mtei + MeeteiMayek, + + /// Mult + Multani, + + /// Mymr + Myanmar, + + /// Nagm + NagMundari, + + /// Nand + Nandinagari, + + /// Narb + OldNorthArabian, + + /// Nbat + Nabataean, + + /// Newa + Newa, + + /// Nkoo + Nko, + + /// Nshu + Nushu, + + /// Ogam + Ogham, + + /// Olck + OlChiki, + + /// Onao + OlOnal, + + /// Orkh + OldTurkic, + + /// Orya + Oriya, + + /// Osge + Osage, + + /// Osma + Osmanya, + + /// Ougr + OldUyghur, + + /// Palm + Palmyrene, + + /// Pauc + PauCinHau, + + /// Perm + OldPermic, + + /// Phag + PhagsPa, + + /// Phli + InscriptionalPahlavi, + + /// Phlp + PsalterPahlavi, + + /// Phnx + Phoenician, + + /// Plrd + Miao, + + /// Prti + InscriptionalParthian, + + /// Rjng + Rejang, + + /// Rohg + HanifiRohingya, + + /// Runr + Runic, + + /// Samr + Samaritan, + + /// Sarb + OldSouthArabian, + + /// Saur + Saurashtra, + + /// Sgnw + SignWriting, + + /// Shaw + Shavian, + + /// Shrd + Sharada, + + /// Sidd + Siddham, + + /// Sind + Khudawadi, + + /// Sinh + Sinhala, + + /// Sogd + Sogdian, + + /// Sogo + OldSogdian, + + /// Sora + SoraSompeng, + + /// Soyo + Soyombo, + + /// Sund + Sundanese, + + /// Sunu + Sunuwar, + + /// Sylo + SylotiNagri, + + /// Syrc + Syriac, + + /// Tagb + Tagbanwa, + + /// Takr + Takri, + + /// Tale + TaiLe, + + /// Talu + NewTaiLue, + + /// Taml + Tamil, + + /// Tang + Tangut, + + /// Tavt + TaiViet, + + /// Telu + Telugu, + + /// Tfng + Tifinagh, + + /// Tglg + Tagalog, + + /// Thaa + Thaana, + + /// Thai + Thai, + + /// Tibt + Tibetan, + + /// Tirh + Tirhuta, + + /// Tnsa + Tangsa, + + /// Todr + Todhri, + + /// Toto + Toto, + + /// Tutg + TuluTigalari, + + /// Ugar + Ugaritic, + + /// Vaii + Vai, + + /// Vith + Vithkuqi, + + /// Wara + WarangCiti, + + /// Wcho + Wancho, + + /// Xpeo + OldPersian, + + /// Xsux + Cuneiform, + + /// Yezi + Yezidi, + + /// Yiii + Yi, + + /// Zanb + ZanabazarSquare, +} + +internal static partial class UnicodeData +{ + /// Version of the Unicode Character Database the tables were generated from. + public const string Version = "16.0.0"; + + /// ISO 15924 code of each , by its value. + private static readonly string[] ScriptCodes = + [ + "Zzzz", "Zyyy", "Zinh", "Adlm", "Aghb", "Ahom", "Arab", "Armi", "Armn", "Avst", "Bali", "Bamu", "Bass", "Batk", + "Beng", "Bhks", "Bopo", "Brah", "Brai", "Bugi", "Buhd", "Cakm", "Cans", "Cari", "Cham", "Cher", "Chrs", "Copt", + "Cpmn", "Cprt", "Cyrl", "Deva", "Diak", "Dogr", "Dsrt", "Dupl", "Egyp", "Elba", "Elym", "Ethi", "Gara", "Geor", + "Glag", "Gong", "Gonm", "Goth", "Gran", "Grek", "Gujr", "Gukh", "Guru", "Hang", "Hani", "Hano", "Hatr", "Hebr", + "Hira", "Hluw", "Hmng", "Hmnp", "Hrkt", "Hung", "Ital", "Java", "Kali", "Kana", "Kawi", "Khar", "Khmr", "Khoj", + "Kits", "Knda", "Krai", "Kthi", "Lana", "Laoo", "Latn", "Lepc", "Limb", "Lina", "Linb", "Lisu", "Lyci", "Lydi", + "Mahj", "Maka", "Mand", "Mani", "Marc", "Medf", "Mend", "Merc", "Mero", "Mlym", "Modi", "Mong", "Mroo", "Mtei", + "Mult", "Mymr", "Nagm", "Nand", "Narb", "Nbat", "Newa", "Nkoo", "Nshu", "Ogam", "Olck", "Onao", "Orkh", "Orya", + "Osge", "Osma", "Ougr", "Palm", "Pauc", "Perm", "Phag", "Phli", "Phlp", "Phnx", "Plrd", "Prti", "Rjng", "Rohg", + "Runr", "Samr", "Sarb", "Saur", "Sgnw", "Shaw", "Shrd", "Sidd", "Sind", "Sinh", "Sogd", "Sogo", "Sora", "Soyo", + "Sund", "Sunu", "Sylo", "Syrc", "Tagb", "Takr", "Tale", "Talu", "Taml", "Tang", "Tavt", "Telu", "Tfng", "Tglg", + "Thaa", "Thai", "Tibt", "Tirh", "Tnsa", "Todr", "Toto", "Tutg", "Ugar", "Vaii", "Vith", "Wara", "Wcho", "Xpeo", + "Xsux", "Yezi", "Yiii", "Zanb", + ]; + + /// First code point of each range of equal values, ascending. + private static ReadOnlySpan LineBreakStarts => + [ + 0x0000, 0x0009, 0x000A, 0x000B, 0x000D, 0x000E, 0x0020, 0x0021, 0x0022, 0x0023, + 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002A, 0x002B, 0x002C, 0x002D, + 0x002E, 0x002F, 0x0030, 0x003A, 0x003C, 0x003F, 0x0040, 0x005B, 0x005C, 0x005D, + 0x005E, 0x007B, 0x007C, 0x007D, 0x007E, 0x007F, 0x0085, 0x0086, 0x00A0, 0x00A1, + 0x00A2, 0x00A3, 0x00A6, 0x00A7, 0x00A9, 0x00AA, 0x00AB, 0x00AC, 0x00AD, 0x00AE, + 0x00B0, 0x00B1, 0x00B2, 0x00B4, 0x00B5, 0x00B6, 0x00BB, 0x00BC, 0x00BF, 0x00C0, + 0x00D7, 0x00D8, 0x00F7, 0x00F8, 0x02C7, 0x02C8, 0x02C9, 0x02CC, 0x02CD, 0x02CE, + 0x02D0, 0x02D1, 0x02D8, 0x02DC, 0x02DD, 0x02DE, 0x02DF, 0x02E0, 0x0300, 0x034F, + 0x0350, 0x035C, 0x0363, 0x0370, 0x0378, 0x037A, 0x037E, 0x037F, 0x0380, 0x0384, + 0x038B, 0x038C, 0x038D, 0x038E, 0x03A2, 0x03A3, 0x0483, 0x048A, 0x0530, 0x0531, + 0x0557, 0x0559, 0x0589, 0x058A, 0x058B, 0x058D, 0x058F, 0x0590, 0x0591, 0x05BE, + 0x05BF, 0x05C0, 0x05C1, 0x05C3, 0x05C4, 0x05C6, 0x05C7, 0x05C8, 0x05D0, 0x05EB, + 0x05EF, 0x05F3, 0x05F5, 0x0600, 0x0606, 0x0609, 0x060C, 0x060E, 0x0610, 0x061B, + 0x061C, 0x061D, 0x0620, 0x064B, 0x0660, 0x066A, 0x066B, 0x066D, 0x0670, 0x0671, + 0x06D4, 0x06D5, 0x06D6, 0x06DD, 0x06DE, 0x06DF, 0x06E5, 0x06E7, 0x06E9, 0x06EA, + 0x06EE, 0x06F0, 0x06FA, 0x070E, 0x070F, 0x0711, 0x0712, 0x0730, 0x074B, 0x074D, + 0x07A6, 0x07B1, 0x07B2, 0x07C0, 0x07CA, 0x07EB, 0x07F4, 0x07F8, 0x07F9, 0x07FA, + 0x07FB, 0x07FD, 0x07FE, 0x0800, 0x0816, 0x081A, 0x081B, 0x0824, 0x0825, 0x0828, + 0x0829, 0x082E, 0x0830, 0x083F, 0x0840, 0x0859, 0x085C, 0x085E, 0x085F, 0x0860, + 0x086B, 0x0870, 0x088F, 0x0890, 0x0892, 0x0897, 0x08A0, 0x08CA, 0x08E2, 0x08E3, + 0x0904, 0x093A, 0x093D, 0x093E, 0x0950, 0x0951, 0x0958, 0x0962, 0x0964, 0x0966, + 0x0970, 0x0981, 0x0984, 0x0985, 0x098D, 0x098F, 0x0991, 0x0993, 0x09A9, 0x09AA, + 0x09B1, 0x09B2, 0x09B3, 0x09B6, 0x09BA, 0x09BC, 0x09BD, 0x09BE, 0x09C5, 0x09C7, + 0x09C9, 0x09CB, 0x09CE, 0x09CF, 0x09D7, 0x09D8, 0x09DC, 0x09DE, 0x09DF, 0x09E2, + 0x09E4, 0x09E6, 0x09F0, 0x09F2, 0x09F4, 0x09F9, 0x09FA, 0x09FB, 0x09FC, 0x09FE, + 0x09FF, 0x0A01, 0x0A04, 0x0A05, 0x0A0B, 0x0A0F, 0x0A11, 0x0A13, 0x0A29, 0x0A2A, + 0x0A31, 0x0A32, 0x0A34, 0x0A35, 0x0A37, 0x0A38, 0x0A3A, 0x0A3C, 0x0A3D, 0x0A3E, + 0x0A43, 0x0A47, 0x0A49, 0x0A4B, 0x0A4E, 0x0A51, 0x0A52, 0x0A59, 0x0A5D, 0x0A5E, + 0x0A5F, 0x0A66, 0x0A70, 0x0A72, 0x0A75, 0x0A76, 0x0A77, 0x0A81, 0x0A84, 0x0A85, + 0x0A8E, 0x0A8F, 0x0A92, 0x0A93, 0x0AA9, 0x0AAA, 0x0AB1, 0x0AB2, 0x0AB4, 0x0AB5, + 0x0ABA, 0x0ABC, 0x0ABD, 0x0ABE, 0x0AC6, 0x0AC7, 0x0ACA, 0x0ACB, 0x0ACE, 0x0AD0, + 0x0AD1, 0x0AE0, 0x0AE2, 0x0AE4, 0x0AE6, 0x0AF0, 0x0AF1, 0x0AF2, 0x0AF9, 0x0AFA, + 0x0B00, 0x0B01, 0x0B04, 0x0B05, 0x0B0D, 0x0B0F, 0x0B11, 0x0B13, 0x0B29, 0x0B2A, + 0x0B31, 0x0B32, 0x0B34, 0x0B35, 0x0B3A, 0x0B3C, 0x0B3D, 0x0B3E, 0x0B45, 0x0B47, + 0x0B49, 0x0B4B, 0x0B4E, 0x0B55, 0x0B58, 0x0B5C, 0x0B5E, 0x0B5F, 0x0B62, 0x0B64, + 0x0B66, 0x0B70, 0x0B78, 0x0B82, 0x0B83, 0x0B84, 0x0B85, 0x0B8B, 0x0B8E, 0x0B91, + 0x0B92, 0x0B96, 0x0B99, 0x0B9B, 0x0B9C, 0x0B9D, 0x0B9E, 0x0BA0, 0x0BA3, 0x0BA5, + 0x0BA8, 0x0BAB, 0x0BAE, 0x0BBA, 0x0BBE, 0x0BC3, 0x0BC6, 0x0BC9, 0x0BCA, 0x0BCE, + 0x0BD0, 0x0BD1, 0x0BD7, 0x0BD8, 0x0BE6, 0x0BF0, 0x0BF9, 0x0BFA, 0x0BFB, 0x0C00, + 0x0C05, 0x0C0D, 0x0C0E, 0x0C11, 0x0C12, 0x0C29, 0x0C2A, 0x0C3A, 0x0C3C, 0x0C3D, + 0x0C3E, 0x0C45, 0x0C46, 0x0C49, 0x0C4A, 0x0C4E, 0x0C55, 0x0C57, 0x0C58, 0x0C5B, + 0x0C5D, 0x0C5E, 0x0C60, 0x0C62, 0x0C64, 0x0C66, 0x0C70, 0x0C77, 0x0C78, 0x0C81, + 0x0C84, 0x0C85, 0x0C8D, 0x0C8E, 0x0C91, 0x0C92, 0x0CA9, 0x0CAA, 0x0CB4, 0x0CB5, + 0x0CBA, 0x0CBC, 0x0CBD, 0x0CBE, 0x0CC5, 0x0CC6, 0x0CC9, 0x0CCA, 0x0CCE, 0x0CD5, + 0x0CD7, 0x0CDD, 0x0CDF, 0x0CE0, 0x0CE2, 0x0CE4, 0x0CE6, 0x0CF0, 0x0CF1, 0x0CF3, + 0x0CF4, 0x0D00, 0x0D04, 0x0D0D, 0x0D0E, 0x0D11, 0x0D12, 0x0D3B, 0x0D3D, 0x0D3E, + 0x0D45, 0x0D46, 0x0D49, 0x0D4A, 0x0D4E, 0x0D50, 0x0D54, 0x0D57, 0x0D58, 0x0D62, + 0x0D64, 0x0D66, 0x0D70, 0x0D79, 0x0D7A, 0x0D80, 0x0D81, 0x0D84, 0x0D85, 0x0D97, + 0x0D9A, 0x0DB2, 0x0DB3, 0x0DBC, 0x0DBD, 0x0DBE, 0x0DC0, 0x0DC7, 0x0DCA, 0x0DCB, + 0x0DCF, 0x0DD5, 0x0DD6, 0x0DD7, 0x0DD8, 0x0DE0, 0x0DE6, 0x0DF0, 0x0DF2, 0x0DF4, + 0x0DF5, 0x0E01, 0x0E3B, 0x0E3F, 0x0E40, 0x0E4F, 0x0E50, 0x0E5A, 0x0E5C, 0x0E81, + 0x0E83, 0x0E84, 0x0E85, 0x0E86, 0x0E8B, 0x0E8C, 0x0EA4, 0x0EA5, 0x0EA6, 0x0EA7, + 0x0EBE, 0x0EC0, 0x0EC5, 0x0EC6, 0x0EC7, 0x0EC8, 0x0ECF, 0x0ED0, 0x0EDA, 0x0EDC, + 0x0EE0, 0x0F00, 0x0F01, 0x0F05, 0x0F06, 0x0F08, 0x0F09, 0x0F0B, 0x0F0C, 0x0F0D, + 0x0F12, 0x0F13, 0x0F14, 0x0F15, 0x0F18, 0x0F1A, 0x0F20, 0x0F2A, 0x0F34, 0x0F35, + 0x0F36, 0x0F37, 0x0F38, 0x0F39, 0x0F3A, 0x0F3B, 0x0F3C, 0x0F3D, 0x0F3E, 0x0F40, + 0x0F48, 0x0F49, 0x0F6D, 0x0F71, 0x0F7F, 0x0F80, 0x0F85, 0x0F86, 0x0F88, 0x0F8D, + 0x0F98, 0x0F99, 0x0FBD, 0x0FBE, 0x0FC0, 0x0FC6, 0x0FC7, 0x0FCD, 0x0FCE, 0x0FD0, + 0x0FD2, 0x0FD3, 0x0FD4, 0x0FD9, 0x0FDB, 0x1000, 0x1040, 0x104A, 0x104C, 0x1050, + 0x1090, 0x109A, 0x10A0, 0x10C6, 0x10C7, 0x10C8, 0x10CD, 0x10CE, 0x10D0, 0x1100, + 0x1160, 0x11A8, 0x1200, 0x1249, 0x124A, 0x124E, 0x1250, 0x1257, 0x1258, 0x1259, + 0x125A, 0x125E, 0x1260, 0x1289, 0x128A, 0x128E, 0x1290, 0x12B1, 0x12B2, 0x12B6, + 0x12B8, 0x12BF, 0x12C0, 0x12C1, 0x12C2, 0x12C6, 0x12C8, 0x12D7, 0x12D8, 0x1311, + 0x1312, 0x1316, 0x1318, 0x135B, 0x135D, 0x1360, 0x1361, 0x1362, 0x137D, 0x1380, + 0x139A, 0x13A0, 0x13F6, 0x13F8, 0x13FE, 0x1400, 0x1401, 0x1680, 0x1681, 0x169B, + 0x169C, 0x169D, 0x16A0, 0x16EB, 0x16EE, 0x16F9, 0x1700, 0x1712, 0x1716, 0x171F, + 0x1732, 0x1735, 0x1737, 0x1740, 0x1752, 0x1754, 0x1760, 0x176D, 0x176E, 0x1771, + 0x1772, 0x1774, 0x1780, 0x17D4, 0x17D6, 0x17D7, 0x17D8, 0x17D9, 0x17DA, 0x17DB, + 0x17DC, 0x17DE, 0x17E0, 0x17EA, 0x17F0, 0x17FA, 0x1800, 0x1802, 0x1804, 0x1806, + 0x1807, 0x1808, 0x180A, 0x180B, 0x180E, 0x180F, 0x1810, 0x181A, 0x1820, 0x1879, + 0x1880, 0x1885, 0x1887, 0x18A9, 0x18AA, 0x18AB, 0x18B0, 0x18F6, 0x1900, 0x191F, + 0x1920, 0x192C, 0x1930, 0x193C, 0x1940, 0x1941, 0x1944, 0x1946, 0x1950, 0x196E, + 0x1970, 0x1975, 0x1980, 0x19AC, 0x19B0, 0x19CA, 0x19D0, 0x19DB, 0x19DE, 0x19E0, + 0x1A17, 0x1A1C, 0x1A1E, 0x1A20, 0x1A5F, 0x1A60, 0x1A7D, 0x1A7F, 0x1A80, 0x1A8A, + 0x1A90, 0x1A9A, 0x1AA0, 0x1AAE, 0x1AB0, 0x1ACF, 0x1B00, 0x1B05, 0x1B34, 0x1B44, + 0x1B45, 0x1B4D, 0x1B4E, 0x1B50, 0x1B5A, 0x1B5C, 0x1B5D, 0x1B61, 0x1B6B, 0x1B74, + 0x1B7D, 0x1B80, 0x1B83, 0x1BA1, 0x1BAE, 0x1BB0, 0x1BBA, 0x1BC0, 0x1BE6, 0x1BF2, + 0x1BF4, 0x1BFC, 0x1C24, 0x1C38, 0x1C3B, 0x1C40, 0x1C4A, 0x1C4D, 0x1C50, 0x1C5A, + 0x1C7E, 0x1C80, 0x1C8B, 0x1C90, 0x1CBB, 0x1CBD, 0x1CC8, 0x1CD0, 0x1CD3, 0x1CD4, + 0x1CE9, 0x1CED, 0x1CEE, 0x1CF4, 0x1CF5, 0x1CF7, 0x1CFA, 0x1CFB, 0x1D00, 0x1DC0, + 0x1DCD, 0x1DCE, 0x1DFC, 0x1DFD, 0x1E00, 0x1F16, 0x1F18, 0x1F1E, 0x1F20, 0x1F46, + 0x1F48, 0x1F4E, 0x1F50, 0x1F58, 0x1F59, 0x1F5A, 0x1F5B, 0x1F5C, 0x1F5D, 0x1F5E, + 0x1F5F, 0x1F7E, 0x1F80, 0x1FB5, 0x1FB6, 0x1FC5, 0x1FC6, 0x1FD4, 0x1FD6, 0x1FDC, + 0x1FDD, 0x1FF0, 0x1FF2, 0x1FF5, 0x1FF6, 0x1FFD, 0x1FFE, 0x1FFF, 0x2000, 0x2007, + 0x2008, 0x200B, 0x200C, 0x200D, 0x200E, 0x2010, 0x2011, 0x2012, 0x2014, 0x2015, + 0x2017, 0x2018, 0x201A, 0x201B, 0x201E, 0x201F, 0x2020, 0x2022, 0x2024, 0x2027, + 0x2028, 0x202A, 0x202F, 0x2030, 0x2038, 0x2039, 0x203B, 0x203C, 0x203E, 0x2044, + 0x2045, 0x2046, 0x2047, 0x204A, 0x2056, 0x2057, 0x2058, 0x205C, 0x205D, 0x2060, + 0x2061, 0x2065, 0x2066, 0x2070, 0x2072, 0x2074, 0x2075, 0x207D, 0x207E, 0x207F, + 0x2080, 0x2081, 0x2085, 0x208D, 0x208E, 0x208F, 0x2090, 0x209D, 0x20A0, 0x20A7, + 0x20A8, 0x20B6, 0x20B7, 0x20BB, 0x20BC, 0x20BE, 0x20BF, 0x20C0, 0x20C1, 0x20D0, + 0x20F1, 0x2100, 0x2103, 0x2104, 0x2105, 0x2106, 0x2109, 0x210A, 0x2113, 0x2114, + 0x2116, 0x2117, 0x2121, 0x2123, 0x212B, 0x212C, 0x2150, 0x215F, 0x2160, 0x216C, + 0x2170, 0x217A, 0x2189, 0x218A, 0x218C, 0x2190, 0x219A, 0x21D2, 0x21D3, 0x21D4, + 0x21D5, 0x2200, 0x2201, 0x2202, 0x2204, 0x2207, 0x2209, 0x220B, 0x220C, 0x220F, + 0x2210, 0x2211, 0x2212, 0x2214, 0x2215, 0x2216, 0x221A, 0x221B, 0x221D, 0x2221, + 0x2223, 0x2224, 0x2225, 0x2226, 0x2227, 0x222D, 0x222E, 0x222F, 0x2234, 0x2238, + 0x223C, 0x223E, 0x2248, 0x2249, 0x224C, 0x224D, 0x2252, 0x2253, 0x2260, 0x2262, + 0x2264, 0x2268, 0x226A, 0x226C, 0x226E, 0x2270, 0x2282, 0x2284, 0x2286, 0x2288, + 0x2295, 0x2296, 0x2299, 0x229A, 0x22A5, 0x22A6, 0x22BF, 0x22C0, 0x22EF, 0x22F0, + 0x2308, 0x2309, 0x230A, 0x230B, 0x230C, 0x2312, 0x2313, 0x231A, 0x231C, 0x2329, + 0x232A, 0x232B, 0x23F0, 0x23F4, 0x242A, 0x2440, 0x244B, 0x2460, 0x24FF, 0x2500, + 0x254C, 0x2550, 0x2575, 0x2580, 0x2590, 0x2592, 0x2596, 0x25A0, 0x25A2, 0x25A3, + 0x25AA, 0x25B2, 0x25B4, 0x25B6, 0x25B8, 0x25BC, 0x25BE, 0x25C0, 0x25C2, 0x25C6, + 0x25C9, 0x25CB, 0x25CC, 0x25CE, 0x25D2, 0x25E2, 0x25E6, 0x25EF, 0x25F0, 0x2600, + 0x2604, 0x2605, 0x2607, 0x2609, 0x260A, 0x260E, 0x2610, 0x2614, 0x2616, 0x2618, + 0x2619, 0x261A, 0x261D, 0x261E, 0x2620, 0x2639, 0x263C, 0x2640, 0x2641, 0x2642, + 0x2643, 0x2660, 0x2662, 0x2663, 0x2666, 0x2667, 0x2668, 0x2669, 0x266B, 0x266C, + 0x266E, 0x266F, 0x2670, 0x267F, 0x2680, 0x269E, 0x26A0, 0x26BD, 0x26C9, 0x26CD, + 0x26CE, 0x26CF, 0x26D2, 0x26D3, 0x26D5, 0x26D8, 0x26DA, 0x26DC, 0x26DD, 0x26DF, + 0x26E2, 0x26E3, 0x26E4, 0x26E8, 0x26EA, 0x26EB, 0x26F1, 0x26F6, 0x26F7, 0x26F9, + 0x26FA, 0x26FB, 0x26FD, 0x2705, 0x2708, 0x270A, 0x270E, 0x2757, 0x2758, 0x275B, + 0x2761, 0x2762, 0x2764, 0x2765, 0x2768, 0x2769, 0x276A, 0x276B, 0x276C, 0x276D, + 0x276E, 0x276F, 0x2770, 0x2771, 0x2772, 0x2773, 0x2774, 0x2775, 0x2776, 0x2794, + 0x27C5, 0x27C6, 0x27C7, 0x27E6, 0x27E7, 0x27E8, 0x27E9, 0x27EA, 0x27EB, 0x27EC, + 0x27ED, 0x27EE, 0x27EF, 0x27F0, 0x2983, 0x2984, 0x2985, 0x2986, 0x2987, 0x2988, + 0x2989, 0x298A, 0x298B, 0x298C, 0x298D, 0x298E, 0x298F, 0x2990, 0x2991, 0x2992, + 0x2993, 0x2994, 0x2995, 0x2996, 0x2997, 0x2998, 0x2999, 0x29D8, 0x29D9, 0x29DA, + 0x29DB, 0x29DC, 0x29FC, 0x29FD, 0x29FE, 0x2B55, 0x2B5A, 0x2B74, 0x2B76, 0x2B96, + 0x2B97, 0x2CEF, 0x2CF2, 0x2CF4, 0x2CF9, 0x2CFA, 0x2CFD, 0x2CFE, 0x2CFF, 0x2D00, + 0x2D26, 0x2D27, 0x2D28, 0x2D2D, 0x2D2E, 0x2D30, 0x2D68, 0x2D6F, 0x2D70, 0x2D71, + 0x2D7F, 0x2D80, 0x2D97, 0x2DA0, 0x2DA7, 0x2DA8, 0x2DAF, 0x2DB0, 0x2DB7, 0x2DB8, + 0x2DBF, 0x2DC0, 0x2DC7, 0x2DC8, 0x2DCF, 0x2DD0, 0x2DD7, 0x2DD8, 0x2DDF, 0x2DE0, + 0x2E00, 0x2E0E, 0x2E16, 0x2E17, 0x2E18, 0x2E19, 0x2E1A, 0x2E1C, 0x2E1E, 0x2E20, + 0x2E22, 0x2E23, 0x2E24, 0x2E25, 0x2E26, 0x2E27, 0x2E28, 0x2E29, 0x2E2A, 0x2E2E, + 0x2E2F, 0x2E30, 0x2E32, 0x2E33, 0x2E35, 0x2E3A, 0x2E3C, 0x2E3F, 0x2E40, 0x2E42, + 0x2E43, 0x2E4B, 0x2E4C, 0x2E4D, 0x2E4E, 0x2E50, 0x2E53, 0x2E55, 0x2E56, 0x2E57, + 0x2E58, 0x2E59, 0x2E5A, 0x2E5B, 0x2E5C, 0x2E5D, 0x2E5E, 0x2E80, 0x2E9A, 0x2E9B, + 0x2EF4, 0x2F00, 0x2FD6, 0x2FF0, 0x3000, 0x3001, 0x3003, 0x3005, 0x3006, 0x3008, + 0x3009, 0x300A, 0x300B, 0x300C, 0x300D, 0x300E, 0x300F, 0x3010, 0x3011, 0x3012, + 0x3014, 0x3015, 0x3016, 0x3017, 0x3018, 0x3019, 0x301A, 0x301B, 0x301C, 0x301D, + 0x301E, 0x3020, 0x302A, 0x3030, 0x3035, 0x3036, 0x303B, 0x303D, 0x3040, 0x3041, + 0x3042, 0x3043, 0x3044, 0x3045, 0x3046, 0x3047, 0x3048, 0x3049, 0x304A, 0x3063, + 0x3064, 0x3083, 0x3084, 0x3085, 0x3086, 0x3087, 0x3088, 0x308E, 0x308F, 0x3095, + 0x3097, 0x3099, 0x309B, 0x309F, 0x30A0, 0x30A1, 0x30A2, 0x30A3, 0x30A4, 0x30A5, + 0x30A6, 0x30A7, 0x30A8, 0x30A9, 0x30AA, 0x30C3, 0x30C4, 0x30E3, 0x30E4, 0x30E5, + 0x30E6, 0x30E7, 0x30E8, 0x30EE, 0x30EF, 0x30F5, 0x30F7, 0x30FB, 0x30FC, 0x30FD, + 0x30FF, 0x3100, 0x3105, 0x3130, 0x3131, 0x318F, 0x3190, 0x31E6, 0x31EF, 0x31F0, + 0x3200, 0x321F, 0x3220, 0x3248, 0x3250, 0x4DC0, 0x4E00, 0xA015, 0xA016, 0xA48D, + 0xA490, 0xA4C7, 0xA4D0, 0xA4FE, 0xA500, 0xA60D, 0xA60E, 0xA60F, 0xA610, 0xA620, + 0xA62A, 0xA62C, 0xA640, 0xA66F, 0xA673, 0xA674, 0xA67E, 0xA69E, 0xA6A0, 0xA6F0, + 0xA6F2, 0xA6F3, 0xA6F8, 0xA700, 0xA7CE, 0xA7D0, 0xA7D2, 0xA7D3, 0xA7D4, 0xA7D5, + 0xA7DD, 0xA7F2, 0xA802, 0xA803, 0xA806, 0xA807, 0xA80B, 0xA80C, 0xA823, 0xA828, + 0xA82C, 0xA82D, 0xA830, 0xA838, 0xA839, 0xA83A, 0xA840, 0xA874, 0xA876, 0xA878, + 0xA880, 0xA882, 0xA8B4, 0xA8C6, 0xA8CE, 0xA8D0, 0xA8DA, 0xA8E0, 0xA8F2, 0xA8FC, + 0xA8FD, 0xA8FF, 0xA900, 0xA90A, 0xA926, 0xA92E, 0xA930, 0xA947, 0xA954, 0xA95F, + 0xA960, 0xA97D, 0xA980, 0xA984, 0xA9B3, 0xA9C0, 0xA9C1, 0xA9C7, 0xA9CA, 0xA9CE, + 0xA9CF, 0xA9D0, 0xA9DA, 0xA9DE, 0xA9E0, 0xA9F0, 0xA9FA, 0xA9FF, 0xAA00, 0xAA29, + 0xAA37, 0xAA40, 0xAA43, 0xAA44, 0xAA4C, 0xAA4E, 0xAA50, 0xAA5A, 0xAA5C, 0xAA5D, + 0xAA60, 0xAAC3, 0xAADB, 0xAAE0, 0xAAEB, 0xAAF0, 0xAAF2, 0xAAF5, 0xAAF7, 0xAB01, + 0xAB07, 0xAB09, 0xAB0F, 0xAB11, 0xAB17, 0xAB20, 0xAB27, 0xAB28, 0xAB2F, 0xAB30, + 0xAB6C, 0xAB70, 0xABE3, 0xABEB, 0xABEC, 0xABEE, 0xABF0, 0xABFA, 0xAC00, 0xAC01, + 0xAC1C, 0xAC1D, 0xAC38, 0xAC39, 0xAC54, 0xAC55, 0xAC70, 0xAC71, 0xAC8C, 0xAC8D, + 0xACA8, 0xACA9, 0xACC4, 0xACC5, 0xACE0, 0xACE1, 0xACFC, 0xACFD, 0xAD18, 0xAD19, + 0xAD34, 0xAD35, 0xAD50, 0xAD51, 0xAD6C, 0xAD6D, 0xAD88, 0xAD89, 0xADA4, 0xADA5, + 0xADC0, 0xADC1, 0xADDC, 0xADDD, 0xADF8, 0xADF9, 0xAE14, 0xAE15, 0xAE30, 0xAE31, + 0xAE4C, 0xAE4D, 0xAE68, 0xAE69, 0xAE84, 0xAE85, 0xAEA0, 0xAEA1, 0xAEBC, 0xAEBD, + 0xAED8, 0xAED9, 0xAEF4, 0xAEF5, 0xAF10, 0xAF11, 0xAF2C, 0xAF2D, 0xAF48, 0xAF49, + 0xAF64, 0xAF65, 0xAF80, 0xAF81, 0xAF9C, 0xAF9D, 0xAFB8, 0xAFB9, 0xAFD4, 0xAFD5, + 0xAFF0, 0xAFF1, 0xB00C, 0xB00D, 0xB028, 0xB029, 0xB044, 0xB045, 0xB060, 0xB061, + 0xB07C, 0xB07D, 0xB098, 0xB099, 0xB0B4, 0xB0B5, 0xB0D0, 0xB0D1, 0xB0EC, 0xB0ED, + 0xB108, 0xB109, 0xB124, 0xB125, 0xB140, 0xB141, 0xB15C, 0xB15D, 0xB178, 0xB179, + 0xB194, 0xB195, 0xB1B0, 0xB1B1, 0xB1CC, 0xB1CD, 0xB1E8, 0xB1E9, 0xB204, 0xB205, + 0xB220, 0xB221, 0xB23C, 0xB23D, 0xB258, 0xB259, 0xB274, 0xB275, 0xB290, 0xB291, + 0xB2AC, 0xB2AD, 0xB2C8, 0xB2C9, 0xB2E4, 0xB2E5, 0xB300, 0xB301, 0xB31C, 0xB31D, + 0xB338, 0xB339, 0xB354, 0xB355, 0xB370, 0xB371, 0xB38C, 0xB38D, 0xB3A8, 0xB3A9, + 0xB3C4, 0xB3C5, 0xB3E0, 0xB3E1, 0xB3FC, 0xB3FD, 0xB418, 0xB419, 0xB434, 0xB435, + 0xB450, 0xB451, 0xB46C, 0xB46D, 0xB488, 0xB489, 0xB4A4, 0xB4A5, 0xB4C0, 0xB4C1, + 0xB4DC, 0xB4DD, 0xB4F8, 0xB4F9, 0xB514, 0xB515, 0xB530, 0xB531, 0xB54C, 0xB54D, + 0xB568, 0xB569, 0xB584, 0xB585, 0xB5A0, 0xB5A1, 0xB5BC, 0xB5BD, 0xB5D8, 0xB5D9, + 0xB5F4, 0xB5F5, 0xB610, 0xB611, 0xB62C, 0xB62D, 0xB648, 0xB649, 0xB664, 0xB665, + 0xB680, 0xB681, 0xB69C, 0xB69D, 0xB6B8, 0xB6B9, 0xB6D4, 0xB6D5, 0xB6F0, 0xB6F1, + 0xB70C, 0xB70D, 0xB728, 0xB729, 0xB744, 0xB745, 0xB760, 0xB761, 0xB77C, 0xB77D, + 0xB798, 0xB799, 0xB7B4, 0xB7B5, 0xB7D0, 0xB7D1, 0xB7EC, 0xB7ED, 0xB808, 0xB809, + 0xB824, 0xB825, 0xB840, 0xB841, 0xB85C, 0xB85D, 0xB878, 0xB879, 0xB894, 0xB895, + 0xB8B0, 0xB8B1, 0xB8CC, 0xB8CD, 0xB8E8, 0xB8E9, 0xB904, 0xB905, 0xB920, 0xB921, + 0xB93C, 0xB93D, 0xB958, 0xB959, 0xB974, 0xB975, 0xB990, 0xB991, 0xB9AC, 0xB9AD, + 0xB9C8, 0xB9C9, 0xB9E4, 0xB9E5, 0xBA00, 0xBA01, 0xBA1C, 0xBA1D, 0xBA38, 0xBA39, + 0xBA54, 0xBA55, 0xBA70, 0xBA71, 0xBA8C, 0xBA8D, 0xBAA8, 0xBAA9, 0xBAC4, 0xBAC5, + 0xBAE0, 0xBAE1, 0xBAFC, 0xBAFD, 0xBB18, 0xBB19, 0xBB34, 0xBB35, 0xBB50, 0xBB51, + 0xBB6C, 0xBB6D, 0xBB88, 0xBB89, 0xBBA4, 0xBBA5, 0xBBC0, 0xBBC1, 0xBBDC, 0xBBDD, + 0xBBF8, 0xBBF9, 0xBC14, 0xBC15, 0xBC30, 0xBC31, 0xBC4C, 0xBC4D, 0xBC68, 0xBC69, + 0xBC84, 0xBC85, 0xBCA0, 0xBCA1, 0xBCBC, 0xBCBD, 0xBCD8, 0xBCD9, 0xBCF4, 0xBCF5, + 0xBD10, 0xBD11, 0xBD2C, 0xBD2D, 0xBD48, 0xBD49, 0xBD64, 0xBD65, 0xBD80, 0xBD81, + 0xBD9C, 0xBD9D, 0xBDB8, 0xBDB9, 0xBDD4, 0xBDD5, 0xBDF0, 0xBDF1, 0xBE0C, 0xBE0D, + 0xBE28, 0xBE29, 0xBE44, 0xBE45, 0xBE60, 0xBE61, 0xBE7C, 0xBE7D, 0xBE98, 0xBE99, + 0xBEB4, 0xBEB5, 0xBED0, 0xBED1, 0xBEEC, 0xBEED, 0xBF08, 0xBF09, 0xBF24, 0xBF25, + 0xBF40, 0xBF41, 0xBF5C, 0xBF5D, 0xBF78, 0xBF79, 0xBF94, 0xBF95, 0xBFB0, 0xBFB1, + 0xBFCC, 0xBFCD, 0xBFE8, 0xBFE9, 0xC004, 0xC005, 0xC020, 0xC021, 0xC03C, 0xC03D, + 0xC058, 0xC059, 0xC074, 0xC075, 0xC090, 0xC091, 0xC0AC, 0xC0AD, 0xC0C8, 0xC0C9, + 0xC0E4, 0xC0E5, 0xC100, 0xC101, 0xC11C, 0xC11D, 0xC138, 0xC139, 0xC154, 0xC155, + 0xC170, 0xC171, 0xC18C, 0xC18D, 0xC1A8, 0xC1A9, 0xC1C4, 0xC1C5, 0xC1E0, 0xC1E1, + 0xC1FC, 0xC1FD, 0xC218, 0xC219, 0xC234, 0xC235, 0xC250, 0xC251, 0xC26C, 0xC26D, + 0xC288, 0xC289, 0xC2A4, 0xC2A5, 0xC2C0, 0xC2C1, 0xC2DC, 0xC2DD, 0xC2F8, 0xC2F9, + 0xC314, 0xC315, 0xC330, 0xC331, 0xC34C, 0xC34D, 0xC368, 0xC369, 0xC384, 0xC385, + 0xC3A0, 0xC3A1, 0xC3BC, 0xC3BD, 0xC3D8, 0xC3D9, 0xC3F4, 0xC3F5, 0xC410, 0xC411, + 0xC42C, 0xC42D, 0xC448, 0xC449, 0xC464, 0xC465, 0xC480, 0xC481, 0xC49C, 0xC49D, + 0xC4B8, 0xC4B9, 0xC4D4, 0xC4D5, 0xC4F0, 0xC4F1, 0xC50C, 0xC50D, 0xC528, 0xC529, + 0xC544, 0xC545, 0xC560, 0xC561, 0xC57C, 0xC57D, 0xC598, 0xC599, 0xC5B4, 0xC5B5, + 0xC5D0, 0xC5D1, 0xC5EC, 0xC5ED, 0xC608, 0xC609, 0xC624, 0xC625, 0xC640, 0xC641, + 0xC65C, 0xC65D, 0xC678, 0xC679, 0xC694, 0xC695, 0xC6B0, 0xC6B1, 0xC6CC, 0xC6CD, + 0xC6E8, 0xC6E9, 0xC704, 0xC705, 0xC720, 0xC721, 0xC73C, 0xC73D, 0xC758, 0xC759, + 0xC774, 0xC775, 0xC790, 0xC791, 0xC7AC, 0xC7AD, 0xC7C8, 0xC7C9, 0xC7E4, 0xC7E5, + 0xC800, 0xC801, 0xC81C, 0xC81D, 0xC838, 0xC839, 0xC854, 0xC855, 0xC870, 0xC871, + 0xC88C, 0xC88D, 0xC8A8, 0xC8A9, 0xC8C4, 0xC8C5, 0xC8E0, 0xC8E1, 0xC8FC, 0xC8FD, + 0xC918, 0xC919, 0xC934, 0xC935, 0xC950, 0xC951, 0xC96C, 0xC96D, 0xC988, 0xC989, + 0xC9A4, 0xC9A5, 0xC9C0, 0xC9C1, 0xC9DC, 0xC9DD, 0xC9F8, 0xC9F9, 0xCA14, 0xCA15, + 0xCA30, 0xCA31, 0xCA4C, 0xCA4D, 0xCA68, 0xCA69, 0xCA84, 0xCA85, 0xCAA0, 0xCAA1, + 0xCABC, 0xCABD, 0xCAD8, 0xCAD9, 0xCAF4, 0xCAF5, 0xCB10, 0xCB11, 0xCB2C, 0xCB2D, + 0xCB48, 0xCB49, 0xCB64, 0xCB65, 0xCB80, 0xCB81, 0xCB9C, 0xCB9D, 0xCBB8, 0xCBB9, + 0xCBD4, 0xCBD5, 0xCBF0, 0xCBF1, 0xCC0C, 0xCC0D, 0xCC28, 0xCC29, 0xCC44, 0xCC45, + 0xCC60, 0xCC61, 0xCC7C, 0xCC7D, 0xCC98, 0xCC99, 0xCCB4, 0xCCB5, 0xCCD0, 0xCCD1, + 0xCCEC, 0xCCED, 0xCD08, 0xCD09, 0xCD24, 0xCD25, 0xCD40, 0xCD41, 0xCD5C, 0xCD5D, + 0xCD78, 0xCD79, 0xCD94, 0xCD95, 0xCDB0, 0xCDB1, 0xCDCC, 0xCDCD, 0xCDE8, 0xCDE9, + 0xCE04, 0xCE05, 0xCE20, 0xCE21, 0xCE3C, 0xCE3D, 0xCE58, 0xCE59, 0xCE74, 0xCE75, + 0xCE90, 0xCE91, 0xCEAC, 0xCEAD, 0xCEC8, 0xCEC9, 0xCEE4, 0xCEE5, 0xCF00, 0xCF01, + 0xCF1C, 0xCF1D, 0xCF38, 0xCF39, 0xCF54, 0xCF55, 0xCF70, 0xCF71, 0xCF8C, 0xCF8D, + 0xCFA8, 0xCFA9, 0xCFC4, 0xCFC5, 0xCFE0, 0xCFE1, 0xCFFC, 0xCFFD, 0xD018, 0xD019, + 0xD034, 0xD035, 0xD050, 0xD051, 0xD06C, 0xD06D, 0xD088, 0xD089, 0xD0A4, 0xD0A5, + 0xD0C0, 0xD0C1, 0xD0DC, 0xD0DD, 0xD0F8, 0xD0F9, 0xD114, 0xD115, 0xD130, 0xD131, + 0xD14C, 0xD14D, 0xD168, 0xD169, 0xD184, 0xD185, 0xD1A0, 0xD1A1, 0xD1BC, 0xD1BD, + 0xD1D8, 0xD1D9, 0xD1F4, 0xD1F5, 0xD210, 0xD211, 0xD22C, 0xD22D, 0xD248, 0xD249, + 0xD264, 0xD265, 0xD280, 0xD281, 0xD29C, 0xD29D, 0xD2B8, 0xD2B9, 0xD2D4, 0xD2D5, + 0xD2F0, 0xD2F1, 0xD30C, 0xD30D, 0xD328, 0xD329, 0xD344, 0xD345, 0xD360, 0xD361, + 0xD37C, 0xD37D, 0xD398, 0xD399, 0xD3B4, 0xD3B5, 0xD3D0, 0xD3D1, 0xD3EC, 0xD3ED, + 0xD408, 0xD409, 0xD424, 0xD425, 0xD440, 0xD441, 0xD45C, 0xD45D, 0xD478, 0xD479, + 0xD494, 0xD495, 0xD4B0, 0xD4B1, 0xD4CC, 0xD4CD, 0xD4E8, 0xD4E9, 0xD504, 0xD505, + 0xD520, 0xD521, 0xD53C, 0xD53D, 0xD558, 0xD559, 0xD574, 0xD575, 0xD590, 0xD591, + 0xD5AC, 0xD5AD, 0xD5C8, 0xD5C9, 0xD5E4, 0xD5E5, 0xD600, 0xD601, 0xD61C, 0xD61D, + 0xD638, 0xD639, 0xD654, 0xD655, 0xD670, 0xD671, 0xD68C, 0xD68D, 0xD6A8, 0xD6A9, + 0xD6C4, 0xD6C5, 0xD6E0, 0xD6E1, 0xD6FC, 0xD6FD, 0xD718, 0xD719, 0xD734, 0xD735, + 0xD750, 0xD751, 0xD76C, 0xD76D, 0xD788, 0xD789, 0xD7A4, 0xD7B0, 0xD7C7, 0xD7CB, + 0xD7FC, 0xD800, 0xE000, 0xF900, 0xFB00, 0xFB07, 0xFB13, 0xFB18, 0xFB1D, 0xFB1E, + 0xFB1F, 0xFB29, 0xFB2A, 0xFB37, 0xFB38, 0xFB3D, 0xFB3E, 0xFB3F, 0xFB40, 0xFB42, + 0xFB43, 0xFB45, 0xFB46, 0xFB50, 0xFBC3, 0xFBD3, 0xFD3E, 0xFD3F, 0xFD40, 0xFD90, + 0xFD92, 0xFDC8, 0xFDCF, 0xFDD0, 0xFDF0, 0xFDFC, 0xFDFD, 0xFE00, 0xFE10, 0xFE13, + 0xFE15, 0xFE17, 0xFE18, 0xFE19, 0xFE1A, 0xFE20, 0xFE21, 0xFE22, 0xFE23, 0xFE24, + 0xFE25, 0xFE26, 0xFE28, 0xFE29, 0xFE2A, 0xFE2B, 0xFE2C, 0xFE2D, 0xFE2F, 0xFE30, + 0xFE35, 0xFE36, 0xFE37, 0xFE38, 0xFE39, 0xFE3A, 0xFE3B, 0xFE3C, 0xFE3D, 0xFE3E, + 0xFE3F, 0xFE40, 0xFE41, 0xFE42, 0xFE43, 0xFE44, 0xFE45, 0xFE47, 0xFE48, 0xFE49, + 0xFE50, 0xFE51, 0xFE52, 0xFE53, 0xFE54, 0xFE56, 0xFE58, 0xFE59, 0xFE5A, 0xFE5B, + 0xFE5C, 0xFE5D, 0xFE5E, 0xFE5F, 0xFE67, 0xFE68, 0xFE69, 0xFE6A, 0xFE6B, 0xFE6C, + 0xFE70, 0xFE75, 0xFE76, 0xFEFD, 0xFEFF, 0xFF00, 0xFF01, 0xFF02, 0xFF04, 0xFF05, + 0xFF06, 0xFF08, 0xFF09, 0xFF0A, 0xFF0C, 0xFF0D, 0xFF0E, 0xFF0F, 0xFF1A, 0xFF1C, + 0xFF1F, 0xFF20, 0xFF3B, 0xFF3C, 0xFF3D, 0xFF3E, 0xFF5B, 0xFF5C, 0xFF5D, 0xFF5E, + 0xFF5F, 0xFF60, 0xFF62, 0xFF63, 0xFF65, 0xFF66, 0xFF67, 0xFF71, 0xFF9E, 0xFFA0, + 0xFFBF, 0xFFC2, 0xFFC8, 0xFFCA, 0xFFD0, 0xFFD2, 0xFFD8, 0xFFDA, 0xFFDD, 0xFFE0, + 0xFFE1, 0xFFE2, 0xFFE5, 0xFFE7, 0xFFE8, 0xFFEF, 0xFFF9, 0xFFFC, 0xFFFD, 0xFFFE, + 0x10000, 0x1000C, 0x1000D, 0x10027, 0x10028, 0x1003B, 0x1003C, 0x1003E, 0x1003F, 0x1004E, + 0x10050, 0x1005E, 0x10080, 0x100FB, 0x10100, 0x10103, 0x10107, 0x10134, 0x10137, 0x1018F, + 0x10190, 0x1019D, 0x101A0, 0x101A1, 0x101D0, 0x101FD, 0x101FE, 0x10280, 0x1029D, 0x102A0, + 0x102D1, 0x102E0, 0x102E1, 0x102FC, 0x10300, 0x10324, 0x1032D, 0x1034B, 0x10350, 0x10376, + 0x1037B, 0x10380, 0x1039E, 0x1039F, 0x103A0, 0x103C4, 0x103C8, 0x103D0, 0x103D1, 0x103D6, + 0x10400, 0x1049E, 0x104A0, 0x104AA, 0x104B0, 0x104D4, 0x104D8, 0x104FC, 0x10500, 0x10528, + 0x10530, 0x10564, 0x1056F, 0x1057B, 0x1057C, 0x1058B, 0x1058C, 0x10593, 0x10594, 0x10596, + 0x10597, 0x105A2, 0x105A3, 0x105B2, 0x105B3, 0x105BA, 0x105BB, 0x105BD, 0x105C0, 0x105F4, + 0x10600, 0x10737, 0x10740, 0x10756, 0x10760, 0x10768, 0x10780, 0x10786, 0x10787, 0x107B1, + 0x107B2, 0x107BB, 0x10800, 0x10806, 0x10808, 0x10809, 0x1080A, 0x10836, 0x10837, 0x10839, + 0x1083C, 0x1083D, 0x1083F, 0x10856, 0x10857, 0x10858, 0x1089F, 0x108A7, 0x108B0, 0x108E0, + 0x108F3, 0x108F4, 0x108F6, 0x108FB, 0x1091C, 0x1091F, 0x10920, 0x1093A, 0x1093F, 0x10940, + 0x10980, 0x109B8, 0x109BC, 0x109D0, 0x109D2, 0x10A01, 0x10A04, 0x10A05, 0x10A07, 0x10A0C, + 0x10A10, 0x10A14, 0x10A15, 0x10A18, 0x10A19, 0x10A36, 0x10A38, 0x10A3B, 0x10A3F, 0x10A40, + 0x10A49, 0x10A50, 0x10A58, 0x10A59, 0x10A60, 0x10AA0, 0x10AC0, 0x10AE5, 0x10AE7, 0x10AEB, + 0x10AF0, 0x10AF6, 0x10AF7, 0x10B00, 0x10B36, 0x10B39, 0x10B40, 0x10B56, 0x10B58, 0x10B73, + 0x10B78, 0x10B92, 0x10B99, 0x10B9D, 0x10BA9, 0x10BB0, 0x10C00, 0x10C49, 0x10C80, 0x10CB3, + 0x10CC0, 0x10CF3, 0x10CFA, 0x10D24, 0x10D28, 0x10D30, 0x10D3A, 0x10D40, 0x10D4A, 0x10D66, + 0x10D69, 0x10D6E, 0x10D6F, 0x10D86, 0x10D8E, 0x10D90, 0x10E60, 0x10E7F, 0x10E80, 0x10EAA, + 0x10EAB, 0x10EAD, 0x10EAE, 0x10EB0, 0x10EB2, 0x10EC2, 0x10EC5, 0x10EFC, 0x10F00, 0x10F28, + 0x10F30, 0x10F46, 0x10F51, 0x10F5A, 0x10F70, 0x10F82, 0x10F86, 0x10F8A, 0x10FB0, 0x10FCC, + 0x10FE0, 0x10FF7, 0x11000, 0x11003, 0x11005, 0x11038, 0x11046, 0x11047, 0x11049, 0x1104E, + 0x11052, 0x11066, 0x11070, 0x11071, 0x11073, 0x11075, 0x11076, 0x1107F, 0x11080, 0x11083, + 0x110B0, 0x110BB, 0x110BD, 0x110BE, 0x110C2, 0x110C3, 0x110CD, 0x110CE, 0x110D0, 0x110E9, + 0x110F0, 0x110FA, 0x11100, 0x11103, 0x11127, 0x11135, 0x11136, 0x11140, 0x11144, 0x11145, + 0x11147, 0x11148, 0x11150, 0x11173, 0x11174, 0x11175, 0x11176, 0x11177, 0x11180, 0x11183, + 0x111B3, 0x111C1, 0x111C5, 0x111C7, 0x111C8, 0x111C9, 0x111CD, 0x111CE, 0x111D0, 0x111DA, + 0x111DB, 0x111DC, 0x111DD, 0x111E0, 0x111E1, 0x111F5, 0x11200, 0x11212, 0x11213, 0x1122C, + 0x11238, 0x1123A, 0x1123B, 0x1123D, 0x1123E, 0x1123F, 0x11241, 0x11242, 0x11280, 0x11287, + 0x11288, 0x11289, 0x1128A, 0x1128E, 0x1128F, 0x1129E, 0x1129F, 0x112A9, 0x112AA, 0x112B0, + 0x112DF, 0x112EB, 0x112F0, 0x112FA, 0x11300, 0x11304, 0x11305, 0x1130D, 0x1130F, 0x11311, + 0x11313, 0x11329, 0x1132A, 0x11331, 0x11332, 0x11334, 0x11335, 0x1133A, 0x1133B, 0x1133D, + 0x1133E, 0x11345, 0x11347, 0x11349, 0x1134B, 0x1134D, 0x1134E, 0x11350, 0x11351, 0x11357, + 0x11358, 0x1135D, 0x1135E, 0x11360, 0x11362, 0x11364, 0x11366, 0x1136D, 0x11370, 0x11375, + 0x11380, 0x1138A, 0x1138B, 0x1138C, 0x1138E, 0x1138F, 0x11390, 0x11392, 0x113B6, 0x113B7, + 0x113B8, 0x113C1, 0x113C2, 0x113C3, 0x113C5, 0x113C6, 0x113C7, 0x113CB, 0x113CC, 0x113D0, + 0x113D1, 0x113D2, 0x113D3, 0x113D6, 0x113D7, 0x113D9, 0x113E1, 0x113E3, 0x11400, 0x11435, + 0x11447, 0x1144B, 0x1144F, 0x11450, 0x1145A, 0x1145C, 0x1145D, 0x1145E, 0x1145F, 0x11462, + 0x11480, 0x114B0, 0x114C4, 0x114C8, 0x114D0, 0x114DA, 0x11580, 0x115AF, 0x115B6, 0x115B8, + 0x115C1, 0x115C2, 0x115C4, 0x115C6, 0x115C9, 0x115D8, 0x115DC, 0x115DE, 0x11600, 0x11630, + 0x11641, 0x11643, 0x11645, 0x11650, 0x1165A, 0x11660, 0x1166D, 0x11680, 0x116AB, 0x116B8, + 0x116BA, 0x116C0, 0x116CA, 0x116D0, 0x116E4, 0x11700, 0x1171B, 0x1171D, 0x1172C, 0x11730, + 0x1173A, 0x1173C, 0x1173F, 0x11747, 0x11800, 0x1182C, 0x1183B, 0x1183C, 0x118A0, 0x118E0, + 0x118EA, 0x118F3, 0x118FF, 0x11900, 0x11907, 0x11909, 0x1190A, 0x1190C, 0x11914, 0x11915, + 0x11917, 0x11918, 0x11930, 0x11936, 0x11937, 0x11939, 0x1193B, 0x1193E, 0x1193F, 0x11940, + 0x11941, 0x11942, 0x11944, 0x11947, 0x11950, 0x1195A, 0x119A0, 0x119A8, 0x119AA, 0x119D1, + 0x119D8, 0x119DA, 0x119E1, 0x119E2, 0x119E3, 0x119E4, 0x119E5, 0x11A00, 0x11A01, 0x11A0B, + 0x11A33, 0x11A3A, 0x11A3B, 0x11A3F, 0x11A40, 0x11A41, 0x11A45, 0x11A46, 0x11A47, 0x11A48, + 0x11A50, 0x11A51, 0x11A5C, 0x11A8A, 0x11A9A, 0x11A9D, 0x11A9E, 0x11AA1, 0x11AA3, 0x11AB0, + 0x11AF9, 0x11B00, 0x11B0A, 0x11BC0, 0x11BE2, 0x11BF0, 0x11BFA, 0x11C00, 0x11C09, 0x11C0A, + 0x11C2F, 0x11C37, 0x11C38, 0x11C40, 0x11C41, 0x11C46, 0x11C50, 0x11C5A, 0x11C6D, 0x11C70, + 0x11C71, 0x11C72, 0x11C90, 0x11C92, 0x11CA8, 0x11CA9, 0x11CB7, 0x11D00, 0x11D07, 0x11D08, + 0x11D0A, 0x11D0B, 0x11D31, 0x11D37, 0x11D3A, 0x11D3B, 0x11D3C, 0x11D3E, 0x11D3F, 0x11D46, + 0x11D47, 0x11D48, 0x11D50, 0x11D5A, 0x11D60, 0x11D66, 0x11D67, 0x11D69, 0x11D6A, 0x11D8A, + 0x11D8F, 0x11D90, 0x11D92, 0x11D93, 0x11D98, 0x11D99, 0x11DA0, 0x11DAA, 0x11EE0, 0x11EF2, + 0x11EF3, 0x11EF7, 0x11EF9, 0x11F00, 0x11F02, 0x11F03, 0x11F04, 0x11F11, 0x11F12, 0x11F34, + 0x11F3B, 0x11F3E, 0x11F42, 0x11F43, 0x11F45, 0x11F50, 0x11F5A, 0x11F5B, 0x11FB0, 0x11FB1, + 0x11FC0, 0x11FDD, 0x11FE1, 0x11FF2, 0x11FFF, 0x12000, 0x1239A, 0x12400, 0x1246F, 0x12470, + 0x12475, 0x12480, 0x12544, 0x12F90, 0x12FF3, 0x13000, 0x13258, 0x1325B, 0x1325E, 0x13282, + 0x13283, 0x13286, 0x13287, 0x13288, 0x13289, 0x1328A, 0x13379, 0x1337A, 0x1337C, 0x1342F, + 0x13430, 0x13437, 0x13438, 0x13439, 0x1343C, 0x1343D, 0x1343E, 0x1343F, 0x13440, 0x13441, + 0x13447, 0x13456, 0x13460, 0x143FB, 0x14400, 0x145CE, 0x145CF, 0x145D0, 0x14647, 0x16100, + 0x1611E, 0x16130, 0x1613A, 0x16800, 0x16A39, 0x16A40, 0x16A5F, 0x16A60, 0x16A6A, 0x16A6E, + 0x16A70, 0x16ABF, 0x16AC0, 0x16ACA, 0x16AD0, 0x16AEE, 0x16AF0, 0x16AF5, 0x16AF6, 0x16B00, + 0x16B30, 0x16B37, 0x16B3A, 0x16B44, 0x16B45, 0x16B46, 0x16B50, 0x16B5A, 0x16B5B, 0x16B62, + 0x16B63, 0x16B78, 0x16B7D, 0x16B90, 0x16D40, 0x16D6E, 0x16D70, 0x16D7A, 0x16E40, 0x16E97, + 0x16E99, 0x16E9B, 0x16F00, 0x16F4B, 0x16F4F, 0x16F50, 0x16F51, 0x16F88, 0x16F8F, 0x16F93, + 0x16FA0, 0x16FE0, 0x16FE4, 0x16FE5, 0x16FF0, 0x16FF2, 0x17000, 0x187F8, 0x18800, 0x18B00, + 0x18CD6, 0x18CFF, 0x18D00, 0x18D09, 0x1AFF0, 0x1AFF4, 0x1AFF5, 0x1AFFC, 0x1AFFD, 0x1AFFF, + 0x1B000, 0x1B123, 0x1B132, 0x1B133, 0x1B150, 0x1B153, 0x1B155, 0x1B156, 0x1B164, 0x1B168, + 0x1B170, 0x1B2FC, 0x1BC00, 0x1BC6B, 0x1BC70, 0x1BC7D, 0x1BC80, 0x1BC89, 0x1BC90, 0x1BC9A, + 0x1BC9C, 0x1BC9D, 0x1BC9F, 0x1BCA0, 0x1BCA4, 0x1CC00, 0x1CCF0, 0x1CCFA, 0x1CD00, 0x1CEB4, + 0x1CF00, 0x1CF2E, 0x1CF30, 0x1CF47, 0x1CF50, 0x1CFC4, 0x1D000, 0x1D0F6, 0x1D100, 0x1D127, + 0x1D129, 0x1D165, 0x1D16A, 0x1D16D, 0x1D183, 0x1D185, 0x1D18C, 0x1D1AA, 0x1D1AE, 0x1D1EB, + 0x1D200, 0x1D242, 0x1D245, 0x1D246, 0x1D2C0, 0x1D2D4, 0x1D2E0, 0x1D2F4, 0x1D300, 0x1D357, + 0x1D360, 0x1D379, 0x1D400, 0x1D455, 0x1D456, 0x1D49D, 0x1D49E, 0x1D4A0, 0x1D4A2, 0x1D4A3, + 0x1D4A5, 0x1D4A7, 0x1D4A9, 0x1D4AD, 0x1D4AE, 0x1D4BA, 0x1D4BB, 0x1D4BC, 0x1D4BD, 0x1D4C4, + 0x1D4C5, 0x1D506, 0x1D507, 0x1D50B, 0x1D50D, 0x1D515, 0x1D516, 0x1D51D, 0x1D51E, 0x1D53A, + 0x1D53B, 0x1D53F, 0x1D540, 0x1D545, 0x1D546, 0x1D547, 0x1D54A, 0x1D551, 0x1D552, 0x1D6A6, + 0x1D6A8, 0x1D7CC, 0x1D7CE, 0x1D800, 0x1DA00, 0x1DA37, 0x1DA3B, 0x1DA6D, 0x1DA75, 0x1DA76, + 0x1DA84, 0x1DA85, 0x1DA87, 0x1DA8B, 0x1DA8C, 0x1DA9B, 0x1DAA0, 0x1DAA1, 0x1DAB0, 0x1DF00, + 0x1DF1F, 0x1DF25, 0x1DF2B, 0x1E000, 0x1E007, 0x1E008, 0x1E019, 0x1E01B, 0x1E022, 0x1E023, + 0x1E025, 0x1E026, 0x1E02B, 0x1E030, 0x1E06E, 0x1E08F, 0x1E090, 0x1E100, 0x1E12D, 0x1E130, + 0x1E137, 0x1E13E, 0x1E140, 0x1E14A, 0x1E14E, 0x1E150, 0x1E290, 0x1E2AE, 0x1E2AF, 0x1E2C0, + 0x1E2EC, 0x1E2F0, 0x1E2FA, 0x1E2FF, 0x1E300, 0x1E4D0, 0x1E4EC, 0x1E4F0, 0x1E4FA, 0x1E5D0, + 0x1E5EE, 0x1E5F0, 0x1E5F1, 0x1E5FB, 0x1E5FF, 0x1E600, 0x1E7E0, 0x1E7E7, 0x1E7E8, 0x1E7EC, + 0x1E7ED, 0x1E7EF, 0x1E7F0, 0x1E7FF, 0x1E800, 0x1E8C5, 0x1E8C7, 0x1E8D0, 0x1E8D7, 0x1E900, + 0x1E944, 0x1E94B, 0x1E94C, 0x1E950, 0x1E95A, 0x1E95E, 0x1E960, 0x1EC71, 0x1ECAC, 0x1ECAD, + 0x1ECB0, 0x1ECB1, 0x1ECB5, 0x1ED01, 0x1ED3E, 0x1EE00, 0x1EE04, 0x1EE05, 0x1EE20, 0x1EE21, + 0x1EE23, 0x1EE24, 0x1EE25, 0x1EE27, 0x1EE28, 0x1EE29, 0x1EE33, 0x1EE34, 0x1EE38, 0x1EE39, + 0x1EE3A, 0x1EE3B, 0x1EE3C, 0x1EE42, 0x1EE43, 0x1EE47, 0x1EE48, 0x1EE49, 0x1EE4A, 0x1EE4B, + 0x1EE4C, 0x1EE4D, 0x1EE50, 0x1EE51, 0x1EE53, 0x1EE54, 0x1EE55, 0x1EE57, 0x1EE58, 0x1EE59, + 0x1EE5A, 0x1EE5B, 0x1EE5C, 0x1EE5D, 0x1EE5E, 0x1EE5F, 0x1EE60, 0x1EE61, 0x1EE63, 0x1EE64, + 0x1EE65, 0x1EE67, 0x1EE6B, 0x1EE6C, 0x1EE73, 0x1EE74, 0x1EE78, 0x1EE79, 0x1EE7D, 0x1EE7E, + 0x1EE7F, 0x1EE80, 0x1EE8A, 0x1EE8B, 0x1EE9C, 0x1EEA1, 0x1EEA4, 0x1EEA5, 0x1EEAA, 0x1EEAB, + 0x1EEBC, 0x1EEF0, 0x1EEF2, 0x1F000, 0x1F100, 0x1F10D, 0x1F110, 0x1F12E, 0x1F130, 0x1F16A, + 0x1F170, 0x1F1AD, 0x1F1AE, 0x1F1E6, 0x1F200, 0x1F385, 0x1F386, 0x1F39C, 0x1F39E, 0x1F3B5, + 0x1F3B7, 0x1F3BC, 0x1F3BD, 0x1F3C2, 0x1F3C5, 0x1F3C7, 0x1F3C8, 0x1F3CA, 0x1F3CD, 0x1F3FB, + 0x1F400, 0x1F442, 0x1F444, 0x1F446, 0x1F451, 0x1F466, 0x1F479, 0x1F47C, 0x1F47D, 0x1F481, + 0x1F484, 0x1F485, 0x1F488, 0x1F48F, 0x1F490, 0x1F491, 0x1F492, 0x1F4A0, 0x1F4A1, 0x1F4A2, + 0x1F4A3, 0x1F4A4, 0x1F4A5, 0x1F4AA, 0x1F4AB, 0x1F4AF, 0x1F4B0, 0x1F4B1, 0x1F4B3, 0x1F500, + 0x1F507, 0x1F517, 0x1F525, 0x1F532, 0x1F54A, 0x1F574, 0x1F576, 0x1F57A, 0x1F57B, 0x1F590, + 0x1F591, 0x1F595, 0x1F597, 0x1F5D4, 0x1F5DC, 0x1F5F4, 0x1F5FA, 0x1F645, 0x1F648, 0x1F64B, + 0x1F650, 0x1F676, 0x1F679, 0x1F67C, 0x1F680, 0x1F6A3, 0x1F6A4, 0x1F6B4, 0x1F6B7, 0x1F6C0, + 0x1F6C1, 0x1F6CC, 0x1F6CD, 0x1F700, 0x1F774, 0x1F780, 0x1F7D5, 0x1F800, 0x1F80C, 0x1F810, + 0x1F848, 0x1F850, 0x1F85A, 0x1F860, 0x1F888, 0x1F890, 0x1F8AE, 0x1F8B0, 0x1F8BC, 0x1F8C0, + 0x1F8C2, 0x1F900, 0x1F90C, 0x1F90D, 0x1F90F, 0x1F910, 0x1F918, 0x1F920, 0x1F926, 0x1F927, + 0x1F930, 0x1F93A, 0x1F93C, 0x1F93F, 0x1F977, 0x1F978, 0x1F9B5, 0x1F9B7, 0x1F9B8, 0x1F9BA, + 0x1F9BB, 0x1F9BC, 0x1F9CD, 0x1F9D0, 0x1F9D1, 0x1F9DE, 0x1FA00, 0x1FA54, 0x1FAC3, 0x1FAC6, + 0x1FAF0, 0x1FAF9, 0x1FB00, 0x1FB93, 0x1FB94, 0x1FBF0, 0x1FBFA, 0x1FC00, 0x1FFFE, 0x20000, + 0x2FFFE, 0x30000, 0x3FFFE, 0xE0001, 0xE0002, 0xE0020, 0xE0080, 0xE0100, 0xE01F0, + ]; + + /// The value of each range in . + private static ReadOnlySpan LineBreakValues => + [ + 13, 7, 30, 9, 15, 13, 41, 18, 37, 3, 36, 35, 3, 37, 34, 14, 3, 36, 26, 23, 26, 42, 33, 26, + 3, 18, 3, 34, 36, 14, 3, 34, 7, 12, 3, 13, 31, 13, 19, 34, 35, 36, 3, 1, 3, 1, 37, 3, + 7, 3, 35, 36, 1, 8, 3, 1, 37, 1, 34, 3, 1, 3, 1, 3, 1, 8, 1, 8, 1, 3, 1, 3, + 1, 3, 1, 3, 8, 3, 13, 19, 13, 19, 13, 3, 0, 3, 26, 3, 0, 3, 0, 3, 0, 3, 0, 3, + 13, 3, 0, 3, 0, 3, 26, 7, 0, 3, 36, 0, 13, 7, 13, 3, 13, 3, 13, 18, 13, 0, 22, 0, + 22, 3, 0, 33, 3, 35, 26, 3, 13, 18, 13, 18, 3, 13, 33, 35, 33, 3, 13, 3, 18, 3, 13, 33, + 3, 13, 3, 13, 3, 13, 3, 33, 3, 0, 3, 13, 3, 13, 0, 3, 13, 3, 0, 33, 3, 13, 3, 26, + 18, 3, 0, 13, 36, 3, 13, 3, 13, 3, 13, 3, 13, 0, 3, 0, 3, 13, 0, 3, 0, 3, 0, 3, + 0, 33, 0, 13, 3, 13, 33, 13, 3, 13, 3, 13, 3, 13, 3, 13, 7, 33, 3, 13, 0, 3, 0, 3, + 0, 3, 0, 3, 0, 3, 0, 3, 0, 13, 3, 13, 0, 13, 0, 13, 3, 0, 13, 0, 3, 0, 3, 13, + 0, 33, 3, 35, 3, 35, 3, 36, 3, 13, 0, 13, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, + 0, 3, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 3, 0, 3, 0, 33, 13, 3, 13, 3, 0, 13, + 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 13, 3, 13, 0, 13, 0, 13, 0, 3, 0, 3, + 13, 0, 33, 3, 36, 0, 3, 13, 0, 13, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 13, + 3, 13, 0, 13, 0, 13, 0, 13, 0, 3, 0, 3, 13, 0, 33, 3, 0, 13, 3, 0, 3, 0, 3, 0, + 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 13, 0, 13, 0, 13, 0, 3, 0, 13, 0, + 33, 3, 36, 3, 0, 13, 3, 0, 3, 0, 3, 0, 3, 0, 13, 3, 13, 0, 13, 0, 13, 0, 13, 0, + 3, 0, 3, 0, 3, 13, 0, 33, 0, 8, 3, 13, 8, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 13, + 3, 13, 0, 13, 0, 13, 0, 13, 0, 3, 0, 3, 13, 0, 33, 0, 3, 13, 0, 13, 3, 0, 3, 0, + 3, 13, 3, 13, 0, 13, 0, 13, 3, 0, 3, 13, 3, 13, 0, 33, 3, 35, 3, 0, 13, 0, 3, 0, + 3, 0, 3, 0, 3, 0, 3, 0, 13, 0, 13, 0, 13, 0, 13, 0, 33, 0, 13, 3, 0, 39, 0, 36, + 39, 3, 33, 7, 0, 39, 0, 39, 0, 39, 0, 39, 0, 39, 0, 39, 0, 39, 0, 39, 0, 39, 0, 33, + 0, 39, 0, 3, 8, 3, 8, 19, 8, 7, 19, 18, 19, 3, 18, 3, 13, 3, 33, 3, 7, 13, 3, 13, + 3, 13, 34, 12, 34, 12, 13, 3, 0, 3, 0, 13, 7, 13, 7, 13, 3, 13, 0, 13, 0, 7, 3, 13, + 3, 0, 3, 8, 7, 8, 3, 19, 0, 39, 33, 7, 3, 39, 33, 39, 3, 0, 3, 0, 3, 0, 3, 27, + 29, 28, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, + 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 13, 3, 7, 3, 0, 3, 0, 3, 0, 3, 0, 7, 3, 7, + 3, 34, 12, 0, 3, 7, 3, 0, 3, 13, 0, 3, 13, 7, 0, 3, 13, 0, 3, 0, 3, 0, 13, 0, + 39, 7, 32, 39, 7, 3, 7, 36, 39, 0, 33, 0, 3, 0, 3, 18, 7, 8, 3, 18, 3, 13, 19, 13, + 33, 0, 3, 0, 3, 13, 3, 13, 3, 0, 3, 0, 3, 0, 13, 0, 13, 0, 3, 0, 18, 33, 39, 0, + 39, 0, 39, 0, 39, 0, 33, 0, 39, 3, 13, 0, 3, 39, 0, 39, 0, 13, 33, 0, 33, 0, 39, 0, + 13, 0, 13, 2, 13, 44, 2, 0, 7, 5, 7, 24, 7, 24, 13, 24, 7, 13, 3, 13, 3, 33, 3, 5, + 13, 43, 0, 3, 13, 0, 7, 33, 0, 3, 33, 3, 7, 3, 0, 3, 0, 3, 0, 13, 3, 13, 3, 13, + 3, 13, 3, 13, 3, 0, 3, 13, 19, 13, 19, 13, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, + 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 8, 3, 0, 7, 19, + 7, 46, 13, 47, 13, 7, 19, 7, 6, 1, 3, 37, 34, 37, 34, 37, 1, 3, 25, 7, 9, 13, 19, 35, + 3, 37, 1, 32, 3, 26, 34, 12, 32, 3, 7, 35, 7, 3, 7, 45, 3, 0, 13, 3, 0, 1, 3, 34, + 12, 1, 3, 1, 3, 34, 12, 0, 3, 0, 36, 35, 36, 35, 36, 35, 36, 35, 36, 35, 36, 13, 0, 3, + 35, 3, 1, 3, 35, 3, 1, 3, 36, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 0, 1, + 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 36, 3, 1, 3, 1, 3, 1, 3, + 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, + 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 25, 3, 34, 12, 34, 12, 3, 1, 3, 24, + 3, 34, 12, 3, 24, 3, 0, 3, 0, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, + 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 24, 3, 1, 3, 1, 3, 1, + 3, 24, 1, 24, 3, 24, 16, 24, 3, 24, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 24, 1, 3, 1, + 3, 1, 3, 24, 3, 1, 3, 24, 1, 24, 3, 24, 1, 24, 1, 24, 1, 24, 1, 24, 3, 1, 3, 1, + 24, 1, 24, 1, 24, 16, 24, 1, 24, 3, 24, 16, 3, 1, 3, 37, 3, 18, 24, 3, 34, 12, 34, 12, + 34, 12, 34, 12, 34, 12, 34, 12, 34, 12, 1, 3, 34, 12, 3, 34, 12, 34, 12, 34, 12, 34, 12, 34, + 12, 3, 34, 12, 34, 12, 34, 12, 34, 12, 34, 12, 34, 12, 34, 12, 34, 12, 34, 12, 34, 12, 34, 12, + 3, 34, 12, 34, 12, 3, 34, 12, 3, 1, 3, 0, 3, 0, 3, 13, 3, 0, 18, 7, 3, 18, 7, 3, + 0, 3, 0, 3, 0, 3, 0, 3, 7, 0, 13, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, + 0, 3, 0, 3, 0, 13, 37, 7, 3, 7, 34, 7, 3, 37, 3, 37, 34, 12, 34, 12, 34, 12, 34, 12, + 7, 18, 3, 7, 3, 7, 3, 6, 7, 3, 7, 34, 7, 3, 7, 3, 7, 3, 18, 34, 14, 34, 14, 34, + 14, 34, 14, 7, 0, 24, 0, 24, 0, 24, 0, 24, 7, 12, 24, 32, 24, 34, 12, 34, 12, 34, 12, 34, + 12, 34, 12, 24, 34, 12, 34, 12, 34, 12, 34, 12, 32, 34, 12, 24, 13, 24, 13, 24, 32, 24, 0, 11, + 24, 11, 24, 11, 24, 11, 24, 11, 24, 11, 24, 11, 24, 11, 24, 11, 24, 11, 24, 11, 0, 13, 32, 24, + 32, 11, 24, 11, 24, 11, 24, 11, 24, 11, 24, 11, 24, 11, 24, 11, 24, 11, 24, 11, 24, 11, 24, 32, + 11, 32, 24, 0, 24, 0, 24, 0, 24, 0, 24, 11, 24, 0, 24, 1, 24, 3, 24, 32, 24, 0, 24, 0, + 3, 7, 3, 7, 18, 7, 3, 33, 3, 0, 3, 13, 3, 13, 3, 13, 3, 13, 3, 7, 0, 3, 0, 3, + 0, 3, 0, 3, 0, 3, 13, 3, 13, 3, 13, 3, 13, 3, 13, 0, 3, 35, 3, 0, 3, 8, 18, 0, + 13, 3, 13, 0, 7, 33, 0, 13, 3, 8, 3, 13, 33, 3, 13, 7, 3, 13, 0, 3, 27, 0, 13, 2, + 13, 44, 24, 7, 24, 0, 7, 5, 0, 24, 39, 33, 39, 0, 5, 13, 0, 7, 13, 7, 13, 0, 5, 0, + 24, 7, 39, 0, 39, 3, 13, 7, 3, 13, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, + 13, 7, 13, 0, 33, 0, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, + 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, + 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, + 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, + 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, + 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, + 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, + 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, + 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, + 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, + 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, + 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, + 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, + 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, + 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, + 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, + 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, + 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, + 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, + 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, + 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, + 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, + 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, + 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, + 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, + 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, + 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, + 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, + 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, + 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, + 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, + 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, + 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, + 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 20, 21, 0, 29, 0, 28, 0, 40, 0, 24, 3, 0, 3, 0, + 22, 13, 22, 3, 22, 0, 22, 0, 22, 0, 22, 0, 22, 0, 22, 3, 0, 3, 12, 34, 3, 0, 3, 0, + 3, 0, 3, 35, 3, 13, 12, 32, 18, 34, 12, 25, 0, 19, 13, 19, 13, 19, 13, 19, 13, 19, 13, 19, + 13, 19, 13, 24, 34, 12, 34, 12, 34, 12, 34, 12, 34, 12, 34, 12, 34, 12, 34, 12, 24, 34, 12, 24, + 12, 24, 12, 0, 32, 18, 24, 34, 12, 34, 12, 34, 12, 24, 0, 24, 36, 35, 24, 0, 3, 0, 3, 0, + 45, 0, 18, 24, 36, 35, 24, 34, 12, 24, 12, 24, 12, 24, 32, 24, 18, 24, 34, 24, 12, 24, 34, 24, + 12, 24, 34, 12, 34, 12, 32, 24, 11, 24, 32, 24, 0, 24, 0, 24, 0, 24, 0, 24, 0, 35, 36, 24, + 36, 0, 3, 0, 13, 10, 1, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 7, 0, + 3, 0, 3, 0, 3, 0, 3, 0, 3, 13, 0, 3, 0, 3, 0, 13, 3, 0, 3, 0, 3, 0, 3, 13, + 0, 3, 0, 7, 3, 0, 3, 7, 3, 0, 3, 0, 33, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, + 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, + 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 7, 3, 0, 3, 0, 3, 0, 3, + 0, 3, 0, 7, 3, 0, 3, 0, 3, 0, 3, 0, 3, 13, 0, 13, 0, 13, 3, 0, 3, 0, 3, 0, + 13, 0, 13, 3, 0, 7, 3, 0, 3, 0, 3, 13, 0, 3, 7, 25, 0, 3, 0, 7, 3, 0, 3, 0, + 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 13, 0, 33, 0, 33, 3, 0, 13, 7, 3, 0, + 3, 0, 3, 0, 3, 0, 13, 7, 0, 3, 0, 3, 0, 13, 3, 0, 3, 13, 3, 0, 3, 13, 3, 0, + 3, 0, 3, 0, 13, 4, 2, 13, 44, 7, 24, 0, 24, 5, 13, 2, 13, 2, 0, 19, 13, 3, 13, 3, + 33, 7, 13, 0, 33, 0, 3, 0, 33, 0, 13, 3, 13, 0, 33, 7, 3, 13, 3, 0, 3, 13, 3, 8, + 3, 0, 13, 3, 13, 3, 7, 3, 7, 13, 3, 13, 33, 3, 8, 3, 7, 0, 3, 0, 3, 0, 3, 13, + 7, 3, 7, 3, 13, 3, 13, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 7, 0, 3, 13, 0, 33, 0, + 13, 0, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, 13, 7, 13, 0, 13, 0, 13, 44, 0, 5, + 0, 13, 0, 7, 5, 2, 13, 0, 13, 0, 13, 0, 5, 0, 5, 0, 5, 0, 5, 2, 0, 24, 13, 0, + 13, 0, 13, 0, 13, 0, 13, 44, 4, 13, 24, 0, 24, 0, 13, 0, 3, 13, 3, 7, 3, 33, 7, 0, + 3, 13, 3, 0, 3, 13, 3, 0, 33, 0, 3, 13, 0, 13, 8, 7, 18, 3, 7, 3, 13, 0, 3, 13, + 7, 3, 0, 33, 0, 8, 0, 3, 13, 3, 0, 33, 0, 33, 0, 39, 0, 39, 0, 33, 39, 7, 39, 0, + 3, 13, 3, 0, 3, 33, 3, 0, 3, 2, 0, 2, 0, 2, 0, 2, 0, 2, 13, 0, 13, 0, 13, 44, + 4, 13, 4, 13, 7, 0, 5, 0, 3, 0, 3, 13, 0, 13, 3, 8, 3, 13, 0, 3, 13, 3, 13, 3, + 13, 8, 3, 7, 8, 3, 13, 0, 3, 13, 3, 13, 7, 3, 8, 7, 0, 3, 0, 8, 0, 3, 0, 33, + 0, 3, 0, 3, 13, 0, 13, 3, 7, 0, 33, 3, 0, 8, 18, 3, 0, 13, 0, 13, 0, 3, 0, 3, + 0, 3, 13, 0, 13, 0, 13, 0, 13, 3, 13, 0, 33, 0, 3, 0, 3, 0, 3, 13, 0, 13, 0, 13, + 3, 0, 33, 0, 5, 7, 13, 7, 0, 13, 4, 13, 2, 0, 2, 13, 0, 13, 44, 7, 24, 5, 13, 0, + 3, 0, 3, 35, 3, 0, 7, 3, 0, 3, 0, 7, 0, 3, 0, 3, 0, 3, 34, 12, 3, 12, 3, 34, + 12, 34, 12, 3, 34, 12, 3, 34, 19, 34, 12, 19, 34, 12, 34, 12, 13, 3, 13, 0, 3, 0, 3, 34, + 12, 3, 0, 5, 13, 5, 0, 3, 0, 3, 0, 33, 0, 7, 3, 0, 33, 0, 3, 0, 13, 7, 0, 3, + 13, 7, 3, 7, 3, 0, 33, 0, 3, 0, 3, 0, 3, 0, 3, 7, 33, 0, 3, 7, 3, 0, 3, 0, + 13, 3, 13, 0, 13, 3, 0, 32, 19, 0, 13, 0, 24, 0, 24, 3, 0, 3, 24, 0, 3, 0, 3, 0, + 3, 0, 24, 0, 11, 0, 11, 0, 11, 0, 11, 0, 24, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 13, + 7, 13, 0, 3, 33, 0, 3, 0, 13, 0, 13, 0, 3, 0, 3, 0, 3, 0, 3, 13, 3, 13, 3, 13, + 3, 13, 3, 0, 3, 13, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, + 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, + 3, 0, 3, 0, 3, 0, 3, 0, 33, 3, 13, 3, 13, 3, 13, 3, 13, 3, 7, 3, 0, 13, 0, 13, + 0, 3, 0, 3, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 3, 0, 13, 0, 3, 0, 13, 3, 0, + 33, 0, 3, 0, 3, 13, 0, 3, 13, 33, 0, 36, 0, 3, 13, 33, 0, 3, 13, 3, 33, 0, 3, 0, + 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 13, 0, 3, 13, 3, 0, 33, 0, 34, 0, 3, 35, 3, + 35, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, + 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, + 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, + 0, 24, 1, 3, 1, 3, 1, 3, 1, 3, 24, 38, 24, 16, 24, 3, 24, 3, 24, 3, 24, 16, 24, 16, + 24, 16, 24, 17, 24, 16, 24, 16, 24, 16, 24, 16, 24, 16, 24, 16, 24, 16, 24, 16, 24, 3, 24, 3, + 24, 3, 24, 16, 24, 3, 24, 3, 24, 3, 24, 3, 24, 3, 24, 16, 24, 16, 24, 16, 24, 16, 24, 3, + 24, 3, 24, 16, 24, 16, 3, 37, 32, 3, 24, 16, 24, 16, 24, 16, 24, 16, 24, 3, 24, 3, 24, 3, + 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 16, 24, 16, 24, 16, 24, 16, 24, 16, 24, + 16, 24, 16, 24, 16, 24, 16, 24, 16, 24, 16, 24, 16, 24, 3, 24, 16, 24, 16, 24, 3, 0, 3, 33, + 0, 24, 0, 24, 0, 24, 0, 13, 0, 13, 0, 13, 0, + ]; + + /// First code point of each range of equal values, ascending. + private static ReadOnlySpan BidiStarts => + [ + 0x0000, 0x0009, 0x000A, 0x000B, 0x000C, 0x000D, 0x000E, 0x001C, 0x001F, 0x0020, + 0x0021, 0x0023, 0x0026, 0x002B, 0x002C, 0x002D, 0x002E, 0x0030, 0x003A, 0x003B, + 0x0041, 0x005B, 0x0061, 0x007B, 0x007F, 0x0085, 0x0086, 0x00A0, 0x00A1, 0x00A2, + 0x00A6, 0x00AA, 0x00AB, 0x00AD, 0x00AE, 0x00B0, 0x00B2, 0x00B4, 0x00B5, 0x00B6, + 0x00B9, 0x00BA, 0x00BB, 0x00C0, 0x00D7, 0x00D8, 0x00F7, 0x00F8, 0x02B9, 0x02BB, + 0x02C2, 0x02D0, 0x02D2, 0x02E0, 0x02E5, 0x02EE, 0x02EF, 0x0300, 0x0370, 0x0374, + 0x0376, 0x037E, 0x037F, 0x0384, 0x0386, 0x0387, 0x0388, 0x03F6, 0x03F7, 0x0483, + 0x048A, 0x058A, 0x058B, 0x058D, 0x058F, 0x0590, 0x0591, 0x05BE, 0x05BF, 0x05C0, + 0x05C1, 0x05C3, 0x05C4, 0x05C6, 0x05C7, 0x05C8, 0x0600, 0x0606, 0x0608, 0x0609, + 0x060B, 0x060C, 0x060D, 0x060E, 0x0610, 0x061B, 0x064B, 0x0660, 0x066A, 0x066B, + 0x066D, 0x0670, 0x0671, 0x06D6, 0x06DD, 0x06DE, 0x06DF, 0x06E5, 0x06E7, 0x06E9, + 0x06EA, 0x06EE, 0x06F0, 0x06FA, 0x0711, 0x0712, 0x0730, 0x074B, 0x07A6, 0x07B1, + 0x07C0, 0x07EB, 0x07F4, 0x07F6, 0x07FA, 0x07FD, 0x07FE, 0x0816, 0x081A, 0x081B, + 0x0824, 0x0825, 0x0828, 0x0829, 0x082E, 0x0859, 0x085C, 0x0860, 0x0890, 0x0892, + 0x0897, 0x08A0, 0x08CA, 0x08E2, 0x08E3, 0x0903, 0x093A, 0x093B, 0x093C, 0x093D, + 0x0941, 0x0949, 0x094D, 0x094E, 0x0951, 0x0958, 0x0962, 0x0964, 0x0981, 0x0982, + 0x09BC, 0x09BD, 0x09C1, 0x09C5, 0x09CD, 0x09CE, 0x09E2, 0x09E4, 0x09F2, 0x09F4, + 0x09FB, 0x09FC, 0x09FE, 0x09FF, 0x0A01, 0x0A03, 0x0A3C, 0x0A3D, 0x0A41, 0x0A43, + 0x0A47, 0x0A49, 0x0A4B, 0x0A4E, 0x0A51, 0x0A52, 0x0A70, 0x0A72, 0x0A75, 0x0A76, + 0x0A81, 0x0A83, 0x0ABC, 0x0ABD, 0x0AC1, 0x0AC6, 0x0AC7, 0x0AC9, 0x0ACD, 0x0ACE, + 0x0AE2, 0x0AE4, 0x0AF1, 0x0AF2, 0x0AFA, 0x0B00, 0x0B01, 0x0B02, 0x0B3C, 0x0B3D, + 0x0B3F, 0x0B40, 0x0B41, 0x0B45, 0x0B4D, 0x0B4E, 0x0B55, 0x0B57, 0x0B62, 0x0B64, + 0x0B82, 0x0B83, 0x0BC0, 0x0BC1, 0x0BCD, 0x0BCE, 0x0BF3, 0x0BF9, 0x0BFA, 0x0BFB, + 0x0C00, 0x0C01, 0x0C04, 0x0C05, 0x0C3C, 0x0C3D, 0x0C3E, 0x0C41, 0x0C46, 0x0C49, + 0x0C4A, 0x0C4E, 0x0C55, 0x0C57, 0x0C62, 0x0C64, 0x0C78, 0x0C7F, 0x0C81, 0x0C82, + 0x0CBC, 0x0CBD, 0x0CCC, 0x0CCE, 0x0CE2, 0x0CE4, 0x0D00, 0x0D02, 0x0D3B, 0x0D3D, + 0x0D41, 0x0D45, 0x0D4D, 0x0D4E, 0x0D62, 0x0D64, 0x0D81, 0x0D82, 0x0DCA, 0x0DCB, + 0x0DD2, 0x0DD5, 0x0DD6, 0x0DD7, 0x0E31, 0x0E32, 0x0E34, 0x0E3B, 0x0E3F, 0x0E40, + 0x0E47, 0x0E4F, 0x0EB1, 0x0EB2, 0x0EB4, 0x0EBD, 0x0EC8, 0x0ECF, 0x0F18, 0x0F1A, + 0x0F35, 0x0F36, 0x0F37, 0x0F38, 0x0F39, 0x0F3A, 0x0F3E, 0x0F71, 0x0F7F, 0x0F80, + 0x0F85, 0x0F86, 0x0F88, 0x0F8D, 0x0F98, 0x0F99, 0x0FBD, 0x0FC6, 0x0FC7, 0x102D, + 0x1031, 0x1032, 0x1038, 0x1039, 0x103B, 0x103D, 0x103F, 0x1058, 0x105A, 0x105E, + 0x1061, 0x1071, 0x1075, 0x1082, 0x1083, 0x1085, 0x1087, 0x108D, 0x108E, 0x109D, + 0x109E, 0x135D, 0x1360, 0x1390, 0x139A, 0x1400, 0x1401, 0x1680, 0x1681, 0x169B, + 0x169D, 0x1712, 0x1715, 0x1732, 0x1734, 0x1752, 0x1754, 0x1772, 0x1774, 0x17B4, + 0x17B6, 0x17B7, 0x17BE, 0x17C6, 0x17C7, 0x17C9, 0x17D4, 0x17DB, 0x17DC, 0x17DD, + 0x17DE, 0x17F0, 0x17FA, 0x1800, 0x180B, 0x180E, 0x180F, 0x1810, 0x1885, 0x1887, + 0x18A9, 0x18AA, 0x1920, 0x1923, 0x1927, 0x1929, 0x1932, 0x1933, 0x1939, 0x193C, + 0x1940, 0x1941, 0x1944, 0x1946, 0x19DE, 0x1A00, 0x1A17, 0x1A19, 0x1A1B, 0x1A1C, + 0x1A56, 0x1A57, 0x1A58, 0x1A5F, 0x1A60, 0x1A61, 0x1A62, 0x1A63, 0x1A65, 0x1A6D, + 0x1A73, 0x1A7D, 0x1A7F, 0x1A80, 0x1AB0, 0x1ACF, 0x1B00, 0x1B04, 0x1B34, 0x1B35, + 0x1B36, 0x1B3B, 0x1B3C, 0x1B3D, 0x1B42, 0x1B43, 0x1B6B, 0x1B74, 0x1B80, 0x1B82, + 0x1BA2, 0x1BA6, 0x1BA8, 0x1BAA, 0x1BAB, 0x1BAE, 0x1BE6, 0x1BE7, 0x1BE8, 0x1BEA, + 0x1BED, 0x1BEE, 0x1BEF, 0x1BF2, 0x1C2C, 0x1C34, 0x1C36, 0x1C38, 0x1CD0, 0x1CD3, + 0x1CD4, 0x1CE1, 0x1CE2, 0x1CE9, 0x1CED, 0x1CEE, 0x1CF4, 0x1CF5, 0x1CF8, 0x1CFA, + 0x1DC0, 0x1E00, 0x1FBD, 0x1FBE, 0x1FBF, 0x1FC2, 0x1FCD, 0x1FD0, 0x1FDD, 0x1FE0, + 0x1FED, 0x1FF0, 0x1FFD, 0x1FFF, 0x2000, 0x200B, 0x200E, 0x200F, 0x2010, 0x2028, + 0x2029, 0x202A, 0x202B, 0x202C, 0x202D, 0x202E, 0x202F, 0x2030, 0x2035, 0x2044, + 0x2045, 0x205F, 0x2060, 0x2066, 0x2067, 0x2068, 0x2069, 0x206A, 0x2070, 0x2071, + 0x2074, 0x207A, 0x207C, 0x207F, 0x2080, 0x208A, 0x208C, 0x208F, 0x20A0, 0x20D0, + 0x20F1, 0x2100, 0x2102, 0x2103, 0x2107, 0x2108, 0x210A, 0x2114, 0x2115, 0x2116, + 0x2119, 0x211E, 0x2124, 0x2125, 0x2126, 0x2127, 0x2128, 0x2129, 0x212A, 0x212E, + 0x212F, 0x213A, 0x213C, 0x2140, 0x2145, 0x214A, 0x214E, 0x2150, 0x2160, 0x2189, + 0x218C, 0x2190, 0x2212, 0x2213, 0x2214, 0x2336, 0x237B, 0x2395, 0x2396, 0x242A, + 0x2440, 0x244B, 0x2460, 0x2488, 0x249C, 0x24EA, 0x26AC, 0x26AD, 0x2800, 0x2900, + 0x2B74, 0x2B76, 0x2B96, 0x2B97, 0x2C00, 0x2CE5, 0x2CEB, 0x2CEF, 0x2CF2, 0x2CF9, + 0x2D00, 0x2D7F, 0x2D80, 0x2DE0, 0x2E00, 0x2E5E, 0x2E80, 0x2E9A, 0x2E9B, 0x2EF4, + 0x2F00, 0x2FD6, 0x2FF0, 0x3000, 0x3001, 0x3005, 0x3008, 0x3021, 0x302A, 0x302E, + 0x3030, 0x3031, 0x3036, 0x3038, 0x303D, 0x3040, 0x3099, 0x309B, 0x309D, 0x30A0, + 0x30A1, 0x30FB, 0x30FC, 0x31C0, 0x31E6, 0x31EF, 0x31F0, 0x321D, 0x321F, 0x3250, + 0x3260, 0x327C, 0x327F, 0x32B1, 0x32C0, 0x32CC, 0x32D0, 0x3377, 0x337B, 0x33DE, + 0x33E0, 0x33FF, 0x3400, 0x4DC0, 0x4E00, 0xA490, 0xA4C7, 0xA60D, 0xA610, 0xA66F, + 0xA673, 0xA674, 0xA67E, 0xA680, 0xA69E, 0xA6A0, 0xA6F0, 0xA6F2, 0xA700, 0xA722, + 0xA788, 0xA789, 0xA802, 0xA803, 0xA806, 0xA807, 0xA80B, 0xA80C, 0xA825, 0xA827, + 0xA828, 0xA82C, 0xA82D, 0xA838, 0xA83A, 0xA874, 0xA878, 0xA8C4, 0xA8C6, 0xA8E0, + 0xA8F2, 0xA8FF, 0xA900, 0xA926, 0xA92E, 0xA947, 0xA952, 0xA980, 0xA983, 0xA9B3, + 0xA9B4, 0xA9B6, 0xA9BA, 0xA9BC, 0xA9BE, 0xA9E5, 0xA9E6, 0xAA29, 0xAA2F, 0xAA31, + 0xAA33, 0xAA35, 0xAA37, 0xAA43, 0xAA44, 0xAA4C, 0xAA4D, 0xAA7C, 0xAA7D, 0xAAB0, + 0xAAB1, 0xAAB2, 0xAAB5, 0xAAB7, 0xAAB9, 0xAABE, 0xAAC0, 0xAAC1, 0xAAC2, 0xAAEC, + 0xAAEE, 0xAAF6, 0xAAF7, 0xAB6A, 0xAB6C, 0xABE5, 0xABE6, 0xABE8, 0xABE9, 0xABED, + 0xABEE, 0xFB1D, 0xFB1E, 0xFB1F, 0xFB29, 0xFB2A, 0xFB50, 0xFD3E, 0xFD50, 0xFDCF, + 0xFDD0, 0xFDF0, 0xFDFD, 0xFE00, 0xFE10, 0xFE1A, 0xFE20, 0xFE30, 0xFE50, 0xFE51, + 0xFE52, 0xFE53, 0xFE54, 0xFE55, 0xFE56, 0xFE5F, 0xFE60, 0xFE62, 0xFE64, 0xFE67, + 0xFE68, 0xFE69, 0xFE6B, 0xFE6C, 0xFE70, 0xFEFF, 0xFF00, 0xFF01, 0xFF03, 0xFF06, + 0xFF0B, 0xFF0C, 0xFF0D, 0xFF0E, 0xFF10, 0xFF1A, 0xFF1B, 0xFF21, 0xFF3B, 0xFF41, + 0xFF5B, 0xFF66, 0xFFE0, 0xFFE2, 0xFFE5, 0xFFE7, 0xFFE8, 0xFFEF, 0xFFF0, 0xFFF9, + 0xFFFE, 0x10000, 0x10101, 0x10102, 0x10140, 0x1018D, 0x10190, 0x1019D, 0x101A0, 0x101A1, + 0x101FD, 0x101FE, 0x102E0, 0x102E1, 0x102FC, 0x10376, 0x1037B, 0x10800, 0x1091F, 0x10920, + 0x10A01, 0x10A04, 0x10A05, 0x10A07, 0x10A0C, 0x10A10, 0x10A38, 0x10A3B, 0x10A3F, 0x10A40, + 0x10AE5, 0x10AE7, 0x10B39, 0x10B40, 0x10D00, 0x10D24, 0x10D28, 0x10D30, 0x10D3A, 0x10D40, + 0x10D4A, 0x10D69, 0x10D6E, 0x10D6F, 0x10E60, 0x10E7F, 0x10EAB, 0x10EAD, 0x10EC0, 0x10EFC, + 0x10F00, 0x10F30, 0x10F46, 0x10F51, 0x10F70, 0x10F82, 0x10F86, 0x11000, 0x11001, 0x11002, + 0x11038, 0x11047, 0x11052, 0x11066, 0x11070, 0x11071, 0x11073, 0x11075, 0x1107F, 0x11082, + 0x110B3, 0x110B7, 0x110B9, 0x110BB, 0x110C2, 0x110C3, 0x11100, 0x11103, 0x11127, 0x1112C, + 0x1112D, 0x11135, 0x11173, 0x11174, 0x11180, 0x11182, 0x111B6, 0x111BF, 0x111C9, 0x111CD, + 0x111CF, 0x111D0, 0x1122F, 0x11232, 0x11234, 0x11235, 0x11236, 0x11238, 0x1123E, 0x1123F, + 0x11241, 0x11242, 0x112DF, 0x112E0, 0x112E3, 0x112EB, 0x11300, 0x11302, 0x1133B, 0x1133D, + 0x11340, 0x11341, 0x11366, 0x1136D, 0x11370, 0x11375, 0x113BB, 0x113C1, 0x113CE, 0x113CF, + 0x113D0, 0x113D1, 0x113D2, 0x113D3, 0x113E1, 0x113E3, 0x11438, 0x11440, 0x11442, 0x11445, + 0x11446, 0x11447, 0x1145E, 0x1145F, 0x114B3, 0x114B9, 0x114BA, 0x114BB, 0x114BF, 0x114C1, + 0x114C2, 0x114C4, 0x115B2, 0x115B6, 0x115BC, 0x115BE, 0x115BF, 0x115C1, 0x115DC, 0x115DE, + 0x11633, 0x1163B, 0x1163D, 0x1163E, 0x1163F, 0x11641, 0x11660, 0x1166D, 0x116AB, 0x116AC, + 0x116AD, 0x116AE, 0x116B0, 0x116B6, 0x116B7, 0x116B8, 0x1171D, 0x1171E, 0x1171F, 0x11720, + 0x11722, 0x11726, 0x11727, 0x1172C, 0x1182F, 0x11838, 0x11839, 0x1183B, 0x1193B, 0x1193D, + 0x1193E, 0x1193F, 0x11943, 0x11944, 0x119D4, 0x119D8, 0x119DA, 0x119DC, 0x119E0, 0x119E1, + 0x11A01, 0x11A07, 0x11A09, 0x11A0B, 0x11A33, 0x11A39, 0x11A3B, 0x11A3F, 0x11A47, 0x11A48, + 0x11A51, 0x11A57, 0x11A59, 0x11A5C, 0x11A8A, 0x11A97, 0x11A98, 0x11A9A, 0x11C30, 0x11C37, + 0x11C38, 0x11C3E, 0x11C92, 0x11CA8, 0x11CAA, 0x11CB1, 0x11CB2, 0x11CB4, 0x11CB5, 0x11CB7, + 0x11D31, 0x11D37, 0x11D3A, 0x11D3B, 0x11D3C, 0x11D3E, 0x11D3F, 0x11D46, 0x11D47, 0x11D48, + 0x11D90, 0x11D92, 0x11D95, 0x11D96, 0x11D97, 0x11D98, 0x11EF3, 0x11EF5, 0x11F00, 0x11F02, + 0x11F36, 0x11F3B, 0x11F40, 0x11F41, 0x11F42, 0x11F43, 0x11F5A, 0x11F5B, 0x11FD5, 0x11FDD, + 0x11FE1, 0x11FF2, 0x13440, 0x13441, 0x13447, 0x13456, 0x1611E, 0x1612A, 0x1612D, 0x16130, + 0x16AF0, 0x16AF5, 0x16B30, 0x16B37, 0x16F4F, 0x16F50, 0x16F8F, 0x16F93, 0x16FE2, 0x16FE3, + 0x16FE4, 0x16FE5, 0x1BC9D, 0x1BC9F, 0x1BCA0, 0x1BCA4, 0x1CC00, 0x1CCD6, 0x1CCF0, 0x1CCFA, + 0x1CD00, 0x1CEB4, 0x1CF00, 0x1CF2E, 0x1CF30, 0x1CF47, 0x1D167, 0x1D16A, 0x1D173, 0x1D17B, + 0x1D183, 0x1D185, 0x1D18C, 0x1D1AA, 0x1D1AE, 0x1D1E9, 0x1D1EB, 0x1D200, 0x1D242, 0x1D245, + 0x1D246, 0x1D300, 0x1D357, 0x1D6C1, 0x1D6C2, 0x1D6DB, 0x1D6DC, 0x1D6FB, 0x1D6FC, 0x1D715, + 0x1D716, 0x1D735, 0x1D736, 0x1D74F, 0x1D750, 0x1D76F, 0x1D770, 0x1D789, 0x1D78A, 0x1D7A9, + 0x1D7AA, 0x1D7C3, 0x1D7C4, 0x1D7CE, 0x1D800, 0x1DA00, 0x1DA37, 0x1DA3B, 0x1DA6D, 0x1DA75, + 0x1DA76, 0x1DA84, 0x1DA85, 0x1DA9B, 0x1DAA0, 0x1DAA1, 0x1DAB0, 0x1E000, 0x1E007, 0x1E008, + 0x1E019, 0x1E01B, 0x1E022, 0x1E023, 0x1E025, 0x1E026, 0x1E02B, 0x1E08F, 0x1E090, 0x1E130, + 0x1E137, 0x1E2AE, 0x1E2AF, 0x1E2EC, 0x1E2F0, 0x1E2FF, 0x1E300, 0x1E4EC, 0x1E4F0, 0x1E5EE, + 0x1E5F0, 0x1E800, 0x1E8D0, 0x1E8D7, 0x1E944, 0x1E94B, 0x1EC70, 0x1ECC0, 0x1ED00, 0x1ED50, + 0x1EE00, 0x1EEF0, 0x1EEF2, 0x1EF00, 0x1F000, 0x1F02C, 0x1F030, 0x1F094, 0x1F0A0, 0x1F0AF, + 0x1F0B1, 0x1F0C0, 0x1F0C1, 0x1F0D0, 0x1F0D1, 0x1F0F6, 0x1F100, 0x1F10B, 0x1F110, 0x1F12F, + 0x1F130, 0x1F16A, 0x1F170, 0x1F1AD, 0x1F1AE, 0x1F260, 0x1F266, 0x1F300, 0x1F6D8, 0x1F6DC, + 0x1F6ED, 0x1F6F0, 0x1F6FD, 0x1F700, 0x1F777, 0x1F77B, 0x1F7DA, 0x1F7E0, 0x1F7EC, 0x1F7F0, + 0x1F7F1, 0x1F800, 0x1F80C, 0x1F810, 0x1F848, 0x1F850, 0x1F85A, 0x1F860, 0x1F888, 0x1F890, + 0x1F8AE, 0x1F8B0, 0x1F8BC, 0x1F8C0, 0x1F8C2, 0x1F900, 0x1FA54, 0x1FA60, 0x1FA6E, 0x1FA70, + 0x1FA7D, 0x1FA80, 0x1FA8A, 0x1FA8F, 0x1FAC7, 0x1FACE, 0x1FADD, 0x1FADF, 0x1FAEA, 0x1FAF0, + 0x1FAF9, 0x1FB00, 0x1FB93, 0x1FB94, 0x1FBF0, 0x1FBFA, 0x1FFFE, 0x20000, 0x2FFFE, 0x30000, + 0x3FFFE, 0x40000, 0x4FFFE, 0x50000, 0x5FFFE, 0x60000, 0x6FFFE, 0x70000, 0x7FFFE, 0x80000, + 0x8FFFE, 0x90000, 0x9FFFE, 0xA0000, 0xAFFFE, 0xB0000, 0xBFFFE, 0xC0000, 0xCFFFE, 0xD0000, + 0xDFFFE, 0xE0100, 0xE01F0, 0xE1000, 0xEFFFE, 0xF0000, 0xFFFFE, 0x100000, 0x10FFFE, + ]; + + /// The value of each range in . + private static ReadOnlySpan BidiValues => + [ + 4, 21, 3, 21, 22, 3, 4, 3, 21, 22, 14, 8, 14, 7, 5, 7, 5, 6, 5, 14, 0, 14, 0, 14, + 4, 3, 4, 5, 14, 8, 14, 0, 14, 4, 14, 8, 6, 14, 0, 14, 6, 0, 14, 0, 14, 0, 14, 0, + 14, 0, 14, 0, 14, 0, 14, 0, 14, 13, 0, 14, 0, 14, 0, 14, 0, 14, 0, 14, 0, 13, 0, 14, + 0, 14, 8, 17, 13, 17, 13, 17, 13, 17, 13, 17, 13, 17, 2, 14, 1, 8, 1, 5, 1, 14, 13, 1, + 13, 2, 8, 2, 1, 13, 1, 13, 2, 14, 13, 1, 13, 14, 13, 1, 6, 1, 13, 1, 13, 1, 13, 1, + 17, 13, 17, 14, 17, 13, 17, 13, 17, 13, 17, 13, 17, 13, 17, 13, 17, 1, 2, 1, 13, 1, 13, 2, + 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, + 8, 0, 8, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, + 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 8, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, + 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 14, 8, 14, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, + 13, 0, 13, 0, 13, 0, 14, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, + 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 8, 0, 13, 0, 13, 0, 13, 0, 13, 0, + 13, 0, 13, 0, 13, 0, 13, 14, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, + 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 14, 0, 14, + 0, 22, 0, 14, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 8, 0, 13, + 0, 14, 0, 14, 13, 4, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 14, 0, 14, 0, + 14, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, + 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, + 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 14, 0, 14, 0, + 14, 0, 14, 0, 14, 0, 14, 0, 22, 4, 0, 17, 14, 22, 3, 10, 18, 15, 12, 20, 5, 8, 14, 5, + 14, 22, 4, 11, 19, 9, 16, 4, 6, 0, 6, 7, 14, 0, 6, 7, 14, 0, 8, 13, 0, 14, 0, 14, + 0, 14, 0, 14, 0, 14, 0, 14, 0, 14, 0, 14, 0, 14, 0, 8, 0, 14, 0, 14, 0, 14, 0, 14, + 0, 14, 0, 14, 7, 8, 14, 0, 14, 0, 14, 0, 14, 0, 14, 6, 0, 14, 0, 14, 0, 14, 0, 14, + 0, 14, 0, 14, 0, 13, 0, 14, 0, 13, 0, 13, 14, 0, 14, 0, 14, 0, 14, 0, 14, 22, 14, 0, + 14, 0, 13, 0, 14, 0, 14, 0, 14, 0, 13, 14, 0, 14, 0, 14, 0, 14, 0, 14, 0, 14, 0, 14, + 0, 14, 0, 14, 0, 14, 0, 14, 0, 14, 0, 14, 0, 14, 0, 14, 0, 14, 0, 13, 14, 13, 14, 0, + 13, 0, 13, 0, 14, 0, 14, 0, 13, 0, 13, 0, 13, 0, 13, 0, 14, 13, 0, 8, 0, 14, 0, 13, + 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, + 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 14, 0, 13, + 0, 13, 0, 13, 0, 17, 13, 17, 7, 17, 1, 14, 1, 14, 4, 1, 14, 13, 14, 0, 13, 14, 5, 14, + 5, 0, 14, 5, 14, 8, 14, 7, 14, 0, 14, 8, 14, 0, 1, 4, 0, 14, 8, 14, 7, 5, 7, 5, + 6, 5, 14, 0, 14, 0, 14, 0, 8, 14, 8, 0, 14, 0, 4, 14, 4, 0, 14, 0, 14, 0, 14, 0, + 14, 0, 13, 0, 13, 6, 0, 13, 0, 17, 14, 17, 13, 17, 13, 17, 13, 17, 13, 17, 13, 17, 13, 17, + 14, 17, 1, 13, 1, 2, 1, 2, 17, 13, 14, 17, 2, 17, 13, 17, 1, 13, 17, 1, 13, 1, 17, 13, + 17, 0, 13, 0, 13, 0, 14, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, + 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, + 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, + 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, + 13, 0, 13, 0, 14, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, + 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, + 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, + 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, + 14, 8, 14, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 14, 0, 13, 0, + 13, 0, 4, 0, 14, 0, 6, 0, 14, 0, 13, 0, 13, 0, 13, 0, 4, 13, 0, 13, 0, 13, 0, 14, + 0, 14, 13, 14, 0, 14, 0, 14, 0, 14, 0, 14, 0, 14, 0, 14, 0, 14, 0, 14, 0, 14, 0, 14, + 0, 14, 0, 6, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, + 0, 13, 0, 13, 0, 13, 0, 13, 0, 13, 0, 8, 0, 13, 0, 13, 0, 17, 13, 17, 13, 17, 1, 17, + 1, 17, 1, 14, 1, 17, 14, 0, 14, 0, 14, 0, 14, 0, 14, 0, 14, 0, 6, 14, 0, 14, 0, 14, + 0, 14, 0, 14, 0, 14, 0, 14, 0, 14, 0, 14, 0, 14, 0, 14, 0, 14, 0, 14, 0, 14, 0, 14, + 0, 14, 0, 14, 0, 14, 0, 14, 0, 14, 0, 14, 0, 14, 0, 14, 0, 14, 0, 14, 0, 14, 0, 14, + 0, 14, 0, 14, 6, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, + 4, 0, 4, 0, 4, 0, 4, 13, 4, 0, 4, 0, 4, 0, 4, + ]; + + /// First code point of each range of equal values, ascending. + private static ReadOnlySpan EastAsianWidthStarts => + [ + 0x0000, 0x0020, 0x007F, 0x00A1, 0x00A2, 0x00A4, 0x00A5, 0x00A7, 0x00A9, 0x00AA, + 0x00AB, 0x00AC, 0x00AD, 0x00AF, 0x00B0, 0x00B5, 0x00B6, 0x00BB, 0x00BC, 0x00C0, + 0x00C6, 0x00C7, 0x00D0, 0x00D1, 0x00D7, 0x00D9, 0x00DE, 0x00E2, 0x00E6, 0x00E7, + 0x00E8, 0x00EB, 0x00EC, 0x00EE, 0x00F0, 0x00F1, 0x00F2, 0x00F4, 0x00F7, 0x00FB, + 0x00FC, 0x00FD, 0x00FE, 0x00FF, 0x0101, 0x0102, 0x0111, 0x0112, 0x0113, 0x0114, + 0x011B, 0x011C, 0x0126, 0x0128, 0x012B, 0x012C, 0x0131, 0x0134, 0x0138, 0x0139, + 0x013F, 0x0143, 0x0144, 0x0145, 0x0148, 0x014C, 0x014D, 0x014E, 0x0152, 0x0154, + 0x0166, 0x0168, 0x016B, 0x016C, 0x01CE, 0x01CF, 0x01D0, 0x01D1, 0x01D2, 0x01D3, + 0x01D4, 0x01D5, 0x01D6, 0x01D7, 0x01D8, 0x01D9, 0x01DA, 0x01DB, 0x01DC, 0x01DD, + 0x0251, 0x0252, 0x0261, 0x0262, 0x02C4, 0x02C5, 0x02C7, 0x02C8, 0x02C9, 0x02CC, + 0x02CD, 0x02CE, 0x02D0, 0x02D1, 0x02D8, 0x02DC, 0x02DD, 0x02DE, 0x02DF, 0x02E0, + 0x0300, 0x0370, 0x0391, 0x03A2, 0x03A3, 0x03AA, 0x03B1, 0x03C2, 0x03C3, 0x03CA, + 0x0401, 0x0402, 0x0410, 0x0450, 0x0451, 0x0452, 0x1100, 0x1160, 0x2010, 0x2011, + 0x2013, 0x2017, 0x2018, 0x201A, 0x201C, 0x201E, 0x2020, 0x2023, 0x2024, 0x2028, + 0x2030, 0x2031, 0x2032, 0x2034, 0x2035, 0x2036, 0x203B, 0x203C, 0x203E, 0x203F, + 0x2074, 0x2075, 0x207F, 0x2080, 0x2081, 0x2085, 0x20A9, 0x20AA, 0x20AC, 0x20AD, + 0x2103, 0x2104, 0x2105, 0x2106, 0x2109, 0x210A, 0x2113, 0x2114, 0x2116, 0x2117, + 0x2121, 0x2123, 0x2126, 0x2127, 0x212B, 0x212C, 0x2153, 0x2155, 0x215B, 0x215F, + 0x2160, 0x216C, 0x2170, 0x217A, 0x2189, 0x218A, 0x2190, 0x219A, 0x21B8, 0x21BA, + 0x21D2, 0x21D3, 0x21D4, 0x21D5, 0x21E7, 0x21E8, 0x2200, 0x2201, 0x2202, 0x2204, + 0x2207, 0x2209, 0x220B, 0x220C, 0x220F, 0x2210, 0x2211, 0x2212, 0x2215, 0x2216, + 0x221A, 0x221B, 0x221D, 0x2221, 0x2223, 0x2224, 0x2225, 0x2226, 0x2227, 0x222D, + 0x222E, 0x222F, 0x2234, 0x2238, 0x223C, 0x223E, 0x2248, 0x2249, 0x224C, 0x224D, + 0x2252, 0x2253, 0x2260, 0x2262, 0x2264, 0x2268, 0x226A, 0x226C, 0x226E, 0x2270, + 0x2282, 0x2284, 0x2286, 0x2288, 0x2295, 0x2296, 0x2299, 0x229A, 0x22A5, 0x22A6, + 0x22BF, 0x22C0, 0x2312, 0x2313, 0x231A, 0x231C, 0x2329, 0x232B, 0x23E9, 0x23ED, + 0x23F0, 0x23F1, 0x23F3, 0x23F4, 0x2460, 0x24EA, 0x24EB, 0x254C, 0x2550, 0x2574, + 0x2580, 0x2590, 0x2592, 0x2596, 0x25A0, 0x25A2, 0x25A3, 0x25AA, 0x25B2, 0x25B4, + 0x25B6, 0x25B8, 0x25BC, 0x25BE, 0x25C0, 0x25C2, 0x25C6, 0x25C9, 0x25CB, 0x25CC, + 0x25CE, 0x25D2, 0x25E2, 0x25E6, 0x25EF, 0x25F0, 0x25FD, 0x25FF, 0x2605, 0x2607, + 0x2609, 0x260A, 0x260E, 0x2610, 0x2614, 0x2616, 0x261C, 0x261D, 0x261E, 0x261F, + 0x2630, 0x2638, 0x2640, 0x2641, 0x2642, 0x2643, 0x2648, 0x2654, 0x2660, 0x2662, + 0x2663, 0x2666, 0x2667, 0x266B, 0x266C, 0x266E, 0x266F, 0x2670, 0x267F, 0x2680, + 0x268A, 0x2690, 0x2693, 0x2694, 0x269E, 0x26A0, 0x26A1, 0x26A2, 0x26AA, 0x26AC, + 0x26BD, 0x26BF, 0x26C0, 0x26C4, 0x26C6, 0x26CE, 0x26CF, 0x26D4, 0x26D5, 0x26E2, + 0x26E3, 0x26E4, 0x26E8, 0x26EA, 0x26EB, 0x26F2, 0x26F4, 0x26F5, 0x26F6, 0x26FA, + 0x26FB, 0x26FD, 0x26FE, 0x2700, 0x2705, 0x2706, 0x270A, 0x270C, 0x2728, 0x2729, + 0x273D, 0x273E, 0x274C, 0x274D, 0x274E, 0x274F, 0x2753, 0x2756, 0x2757, 0x2758, + 0x2776, 0x2780, 0x2795, 0x2798, 0x27B0, 0x27B1, 0x27BF, 0x27C0, 0x27E6, 0x27EE, + 0x2985, 0x2987, 0x2B1B, 0x2B1D, 0x2B50, 0x2B51, 0x2B55, 0x2B56, 0x2B5A, 0x2E80, + 0x2E9A, 0x2E9B, 0x2EF4, 0x2F00, 0x2FD6, 0x2FF0, 0x3000, 0x3001, 0x303F, 0x3041, + 0x3097, 0x3099, 0x3100, 0x3105, 0x3130, 0x3131, 0x318F, 0x3190, 0x31E6, 0x31EF, + 0x321F, 0x3220, 0x3248, 0x3250, 0xA48D, 0xA490, 0xA4C7, 0xA960, 0xA97D, 0xAC00, + 0xD7A4, 0xE000, 0xF900, 0xFB00, 0xFE00, 0xFE10, 0xFE1A, 0xFE30, 0xFE53, 0xFE54, + 0xFE67, 0xFE68, 0xFE6C, 0xFF01, 0xFF61, 0xFFBF, 0xFFC2, 0xFFC8, 0xFFCA, 0xFFD0, + 0xFFD2, 0xFFD8, 0xFFDA, 0xFFDD, 0xFFE0, 0xFFE7, 0xFFE8, 0xFFEF, 0xFFFD, 0xFFFE, + 0x16FE0, 0x16FE5, 0x16FF0, 0x16FF2, 0x17000, 0x187F8, 0x18800, 0x18CD6, 0x18CFF, 0x18D09, + 0x1AFF0, 0x1AFF4, 0x1AFF5, 0x1AFFC, 0x1AFFD, 0x1AFFF, 0x1B000, 0x1B123, 0x1B132, 0x1B133, + 0x1B150, 0x1B153, 0x1B155, 0x1B156, 0x1B164, 0x1B168, 0x1B170, 0x1B2FC, 0x1D300, 0x1D357, + 0x1D360, 0x1D377, 0x1F004, 0x1F005, 0x1F0CF, 0x1F0D0, 0x1F100, 0x1F10B, 0x1F110, 0x1F12E, + 0x1F130, 0x1F16A, 0x1F170, 0x1F18E, 0x1F18F, 0x1F191, 0x1F19B, 0x1F1AD, 0x1F200, 0x1F203, + 0x1F210, 0x1F23C, 0x1F240, 0x1F249, 0x1F250, 0x1F252, 0x1F260, 0x1F266, 0x1F300, 0x1F321, + 0x1F32D, 0x1F336, 0x1F337, 0x1F37D, 0x1F37E, 0x1F394, 0x1F3A0, 0x1F3CB, 0x1F3CF, 0x1F3D4, + 0x1F3E0, 0x1F3F1, 0x1F3F4, 0x1F3F5, 0x1F3F8, 0x1F43F, 0x1F440, 0x1F441, 0x1F442, 0x1F4FD, + 0x1F4FF, 0x1F53E, 0x1F54B, 0x1F54F, 0x1F550, 0x1F568, 0x1F57A, 0x1F57B, 0x1F595, 0x1F597, + 0x1F5A4, 0x1F5A5, 0x1F5FB, 0x1F650, 0x1F680, 0x1F6C6, 0x1F6CC, 0x1F6CD, 0x1F6D0, 0x1F6D3, + 0x1F6D5, 0x1F6D8, 0x1F6DC, 0x1F6E0, 0x1F6EB, 0x1F6ED, 0x1F6F4, 0x1F6FD, 0x1F7E0, 0x1F7EC, + 0x1F7F0, 0x1F7F1, 0x1F90C, 0x1F93B, 0x1F93C, 0x1F946, 0x1F947, 0x1FA00, 0x1FA70, 0x1FA7D, + 0x1FA80, 0x1FA8A, 0x1FA8F, 0x1FAC7, 0x1FACE, 0x1FADD, 0x1FADF, 0x1FAEA, 0x1FAF0, 0x1FAF9, + 0x20000, 0x2FFFE, 0x30000, 0x3FFFE, 0xE0100, 0xE01F0, 0xF0000, 0xFFFFE, 0x100000, 0x10FFFE, + ]; + + /// The value of each range in . + private static ReadOnlySpan EastAsianWidthValues => + [ + 0, 4, 0, 1, 4, 1, 4, 1, 0, 1, 0, 4, 1, 4, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, + 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, + 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, + 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, + 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, + 1, 0, 1, 0, 1, 0, 5, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, + 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 3, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, + 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, + 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, + 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, + 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 5, 0, 5, 0, 5, 0, 5, 0, 5, 0, + 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, + 1, 0, 1, 0, 1, 0, 1, 0, 5, 0, 1, 0, 1, 0, 1, 0, 5, 0, 1, 0, 1, 0, 5, 0, + 1, 0, 1, 0, 5, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 5, 0, 5, 0, 5, 0, 1, 0, + 5, 0, 5, 0, 5, 1, 0, 5, 1, 5, 1, 5, 1, 0, 1, 0, 1, 5, 1, 5, 1, 5, 1, 5, + 1, 5, 1, 0, 5, 0, 5, 0, 5, 0, 1, 0, 5, 0, 5, 0, 5, 0, 5, 0, 1, 0, 5, 0, + 5, 0, 5, 0, 4, 0, 4, 0, 5, 0, 5, 0, 5, 1, 0, 5, 0, 5, 0, 5, 0, 5, 2, 5, + 0, 5, 0, 5, 0, 5, 0, 5, 0, 5, 0, 5, 0, 5, 1, 5, 0, 5, 0, 5, 0, 5, 0, 1, + 5, 0, 1, 5, 0, 5, 0, 5, 0, 5, 0, 2, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 2, 0, + 3, 0, 1, 0, 5, 0, 5, 0, 5, 0, 5, 0, 5, 0, 5, 0, 5, 0, 5, 0, 5, 0, 5, 0, + 5, 0, 5, 0, 5, 0, 5, 0, 5, 0, 5, 0, 5, 0, 5, 0, 1, 0, 1, 0, 1, 0, 1, 5, + 1, 5, 1, 0, 5, 0, 5, 0, 5, 0, 5, 0, 5, 0, 5, 0, 5, 0, 5, 0, 5, 0, 5, 0, + 5, 0, 5, 0, 5, 0, 5, 0, 5, 0, 5, 0, 5, 0, 5, 0, 5, 0, 5, 0, 5, 0, 5, 0, + 5, 0, 5, 0, 5, 0, 5, 0, 5, 0, 5, 0, 5, 0, 5, 0, 5, 0, 5, 0, 5, 0, 5, 0, + 5, 0, 5, 0, 5, 0, 5, 0, 5, 0, 5, 0, 5, 0, 5, 0, 5, 0, 1, 0, 1, 0, 1, 0, + ]; + + /// First code point of each range of equal values, ascending. + private static ReadOnlySpan ScriptStarts => + [ + 0x0000, 0x0041, 0x005B, 0x0061, 0x007B, 0x00AA, 0x00AB, 0x00BA, 0x00BB, 0x00C0, + 0x00D7, 0x00D8, 0x00F7, 0x00F8, 0x02B9, 0x02E0, 0x02E5, 0x02EA, 0x02EC, 0x0300, + 0x0370, 0x0374, 0x0375, 0x0378, 0x037A, 0x037E, 0x037F, 0x0380, 0x0384, 0x0385, + 0x0386, 0x0387, 0x0388, 0x038B, 0x038C, 0x038D, 0x038E, 0x03A2, 0x03A3, 0x03E2, + 0x03F0, 0x0400, 0x0485, 0x0487, 0x0530, 0x0531, 0x0557, 0x0559, 0x058B, 0x058D, + 0x0590, 0x0591, 0x05C8, 0x05D0, 0x05EB, 0x05EF, 0x05F5, 0x0600, 0x0605, 0x0606, + 0x060C, 0x060D, 0x061B, 0x061C, 0x061F, 0x0620, 0x0640, 0x0641, 0x064B, 0x0656, + 0x0670, 0x0671, 0x06DD, 0x06DE, 0x0700, 0x070E, 0x070F, 0x074B, 0x074D, 0x0750, + 0x0780, 0x07B2, 0x07C0, 0x07FB, 0x07FD, 0x0800, 0x082E, 0x0830, 0x083F, 0x0840, + 0x085C, 0x085E, 0x085F, 0x0860, 0x086B, 0x0870, 0x088F, 0x0890, 0x0892, 0x0897, + 0x08E2, 0x08E3, 0x0900, 0x0951, 0x0955, 0x0964, 0x0966, 0x0980, 0x0984, 0x0985, + 0x098D, 0x098F, 0x0991, 0x0993, 0x09A9, 0x09AA, 0x09B1, 0x09B2, 0x09B3, 0x09B6, + 0x09BA, 0x09BC, 0x09C5, 0x09C7, 0x09C9, 0x09CB, 0x09CF, 0x09D7, 0x09D8, 0x09DC, + 0x09DE, 0x09DF, 0x09E4, 0x09E6, 0x09FF, 0x0A01, 0x0A04, 0x0A05, 0x0A0B, 0x0A0F, + 0x0A11, 0x0A13, 0x0A29, 0x0A2A, 0x0A31, 0x0A32, 0x0A34, 0x0A35, 0x0A37, 0x0A38, + 0x0A3A, 0x0A3C, 0x0A3D, 0x0A3E, 0x0A43, 0x0A47, 0x0A49, 0x0A4B, 0x0A4E, 0x0A51, + 0x0A52, 0x0A59, 0x0A5D, 0x0A5E, 0x0A5F, 0x0A66, 0x0A77, 0x0A81, 0x0A84, 0x0A85, + 0x0A8E, 0x0A8F, 0x0A92, 0x0A93, 0x0AA9, 0x0AAA, 0x0AB1, 0x0AB2, 0x0AB4, 0x0AB5, + 0x0ABA, 0x0ABC, 0x0AC6, 0x0AC7, 0x0ACA, 0x0ACB, 0x0ACE, 0x0AD0, 0x0AD1, 0x0AE0, + 0x0AE4, 0x0AE6, 0x0AF2, 0x0AF9, 0x0B00, 0x0B01, 0x0B04, 0x0B05, 0x0B0D, 0x0B0F, + 0x0B11, 0x0B13, 0x0B29, 0x0B2A, 0x0B31, 0x0B32, 0x0B34, 0x0B35, 0x0B3A, 0x0B3C, + 0x0B45, 0x0B47, 0x0B49, 0x0B4B, 0x0B4E, 0x0B55, 0x0B58, 0x0B5C, 0x0B5E, 0x0B5F, + 0x0B64, 0x0B66, 0x0B78, 0x0B82, 0x0B84, 0x0B85, 0x0B8B, 0x0B8E, 0x0B91, 0x0B92, + 0x0B96, 0x0B99, 0x0B9B, 0x0B9C, 0x0B9D, 0x0B9E, 0x0BA0, 0x0BA3, 0x0BA5, 0x0BA8, + 0x0BAB, 0x0BAE, 0x0BBA, 0x0BBE, 0x0BC3, 0x0BC6, 0x0BC9, 0x0BCA, 0x0BCE, 0x0BD0, + 0x0BD1, 0x0BD7, 0x0BD8, 0x0BE6, 0x0BFB, 0x0C00, 0x0C0D, 0x0C0E, 0x0C11, 0x0C12, + 0x0C29, 0x0C2A, 0x0C3A, 0x0C3C, 0x0C45, 0x0C46, 0x0C49, 0x0C4A, 0x0C4E, 0x0C55, + 0x0C57, 0x0C58, 0x0C5B, 0x0C5D, 0x0C5E, 0x0C60, 0x0C64, 0x0C66, 0x0C70, 0x0C77, + 0x0C80, 0x0C8D, 0x0C8E, 0x0C91, 0x0C92, 0x0CA9, 0x0CAA, 0x0CB4, 0x0CB5, 0x0CBA, + 0x0CBC, 0x0CC5, 0x0CC6, 0x0CC9, 0x0CCA, 0x0CCE, 0x0CD5, 0x0CD7, 0x0CDD, 0x0CDF, + 0x0CE0, 0x0CE4, 0x0CE6, 0x0CF0, 0x0CF1, 0x0CF4, 0x0D00, 0x0D0D, 0x0D0E, 0x0D11, + 0x0D12, 0x0D45, 0x0D46, 0x0D49, 0x0D4A, 0x0D50, 0x0D54, 0x0D64, 0x0D66, 0x0D80, + 0x0D81, 0x0D84, 0x0D85, 0x0D97, 0x0D9A, 0x0DB2, 0x0DB3, 0x0DBC, 0x0DBD, 0x0DBE, + 0x0DC0, 0x0DC7, 0x0DCA, 0x0DCB, 0x0DCF, 0x0DD5, 0x0DD6, 0x0DD7, 0x0DD8, 0x0DE0, + 0x0DE6, 0x0DF0, 0x0DF2, 0x0DF5, 0x0E01, 0x0E3B, 0x0E3F, 0x0E40, 0x0E5C, 0x0E81, + 0x0E83, 0x0E84, 0x0E85, 0x0E86, 0x0E8B, 0x0E8C, 0x0EA4, 0x0EA5, 0x0EA6, 0x0EA7, + 0x0EBE, 0x0EC0, 0x0EC5, 0x0EC6, 0x0EC7, 0x0EC8, 0x0ECF, 0x0ED0, 0x0EDA, 0x0EDC, + 0x0EE0, 0x0F00, 0x0F48, 0x0F49, 0x0F6D, 0x0F71, 0x0F98, 0x0F99, 0x0FBD, 0x0FBE, + 0x0FCD, 0x0FCE, 0x0FD5, 0x0FD9, 0x0FDB, 0x1000, 0x10A0, 0x10C6, 0x10C7, 0x10C8, + 0x10CD, 0x10CE, 0x10D0, 0x10FB, 0x10FC, 0x1100, 0x1200, 0x1249, 0x124A, 0x124E, + 0x1250, 0x1257, 0x1258, 0x1259, 0x125A, 0x125E, 0x1260, 0x1289, 0x128A, 0x128E, + 0x1290, 0x12B1, 0x12B2, 0x12B6, 0x12B8, 0x12BF, 0x12C0, 0x12C1, 0x12C2, 0x12C6, + 0x12C8, 0x12D7, 0x12D8, 0x1311, 0x1312, 0x1316, 0x1318, 0x135B, 0x135D, 0x137D, + 0x1380, 0x139A, 0x13A0, 0x13F6, 0x13F8, 0x13FE, 0x1400, 0x1680, 0x169D, 0x16A0, + 0x16EB, 0x16EE, 0x16F9, 0x1700, 0x1716, 0x171F, 0x1720, 0x1735, 0x1737, 0x1740, + 0x1754, 0x1760, 0x176D, 0x176E, 0x1771, 0x1772, 0x1774, 0x1780, 0x17DE, 0x17E0, + 0x17EA, 0x17F0, 0x17FA, 0x1800, 0x1802, 0x1804, 0x1805, 0x1806, 0x181A, 0x1820, + 0x1879, 0x1880, 0x18AB, 0x18B0, 0x18F6, 0x1900, 0x191F, 0x1920, 0x192C, 0x1930, + 0x193C, 0x1940, 0x1941, 0x1944, 0x1950, 0x196E, 0x1970, 0x1975, 0x1980, 0x19AC, + 0x19B0, 0x19CA, 0x19D0, 0x19DB, 0x19DE, 0x19E0, 0x1A00, 0x1A1C, 0x1A1E, 0x1A20, + 0x1A5F, 0x1A60, 0x1A7D, 0x1A7F, 0x1A8A, 0x1A90, 0x1A9A, 0x1AA0, 0x1AAE, 0x1AB0, + 0x1ACF, 0x1B00, 0x1B4D, 0x1B4E, 0x1B80, 0x1BC0, 0x1BF4, 0x1BFC, 0x1C00, 0x1C38, + 0x1C3B, 0x1C4A, 0x1C4D, 0x1C50, 0x1C80, 0x1C8B, 0x1C90, 0x1CBB, 0x1CBD, 0x1CC0, + 0x1CC8, 0x1CD0, 0x1CD3, 0x1CD4, 0x1CE1, 0x1CE2, 0x1CE9, 0x1CED, 0x1CEE, 0x1CF4, + 0x1CF5, 0x1CF8, 0x1CFA, 0x1CFB, 0x1D00, 0x1D26, 0x1D2B, 0x1D2C, 0x1D5D, 0x1D62, + 0x1D66, 0x1D6B, 0x1D78, 0x1D79, 0x1DBF, 0x1DC0, 0x1E00, 0x1F00, 0x1F16, 0x1F18, + 0x1F1E, 0x1F20, 0x1F46, 0x1F48, 0x1F4E, 0x1F50, 0x1F58, 0x1F59, 0x1F5A, 0x1F5B, + 0x1F5C, 0x1F5D, 0x1F5E, 0x1F5F, 0x1F7E, 0x1F80, 0x1FB5, 0x1FB6, 0x1FC5, 0x1FC6, + 0x1FD4, 0x1FD6, 0x1FDC, 0x1FDD, 0x1FF0, 0x1FF2, 0x1FF5, 0x1FF6, 0x1FFF, 0x2000, + 0x200C, 0x200E, 0x2065, 0x2066, 0x2071, 0x2072, 0x2074, 0x207F, 0x2080, 0x208F, + 0x2090, 0x209D, 0x20A0, 0x20C1, 0x20D0, 0x20F1, 0x2100, 0x2126, 0x2127, 0x212A, + 0x212C, 0x2132, 0x2133, 0x214E, 0x214F, 0x2160, 0x2189, 0x218C, 0x2190, 0x242A, + 0x2440, 0x244B, 0x2460, 0x2800, 0x2900, 0x2B74, 0x2B76, 0x2B96, 0x2B97, 0x2C00, + 0x2C60, 0x2C80, 0x2CF4, 0x2CF9, 0x2D00, 0x2D26, 0x2D27, 0x2D28, 0x2D2D, 0x2D2E, + 0x2D30, 0x2D68, 0x2D6F, 0x2D71, 0x2D7F, 0x2D80, 0x2D97, 0x2DA0, 0x2DA7, 0x2DA8, + 0x2DAF, 0x2DB0, 0x2DB7, 0x2DB8, 0x2DBF, 0x2DC0, 0x2DC7, 0x2DC8, 0x2DCF, 0x2DD0, + 0x2DD7, 0x2DD8, 0x2DDF, 0x2DE0, 0x2E00, 0x2E5E, 0x2E80, 0x2E9A, 0x2E9B, 0x2EF4, + 0x2F00, 0x2FD6, 0x2FF0, 0x3005, 0x3006, 0x3007, 0x3008, 0x3021, 0x302A, 0x302E, + 0x3030, 0x3038, 0x303C, 0x3040, 0x3041, 0x3097, 0x3099, 0x309B, 0x309D, 0x30A0, + 0x30A1, 0x30FB, 0x30FD, 0x3100, 0x3105, 0x3130, 0x3131, 0x318F, 0x3190, 0x31A0, + 0x31C0, 0x31E6, 0x31EF, 0x31F0, 0x3200, 0x321F, 0x3220, 0x3260, 0x327F, 0x32D0, + 0x32FF, 0x3300, 0x3358, 0x3400, 0x4DC0, 0x4E00, 0xA000, 0xA48D, 0xA490, 0xA4C7, + 0xA4D0, 0xA500, 0xA62C, 0xA640, 0xA6A0, 0xA6F8, 0xA700, 0xA722, 0xA788, 0xA78B, + 0xA7CE, 0xA7D0, 0xA7D2, 0xA7D3, 0xA7D4, 0xA7D5, 0xA7DD, 0xA7F2, 0xA800, 0xA82D, + 0xA830, 0xA83A, 0xA840, 0xA878, 0xA880, 0xA8C6, 0xA8CE, 0xA8DA, 0xA8E0, 0xA900, + 0xA92E, 0xA92F, 0xA930, 0xA954, 0xA95F, 0xA960, 0xA97D, 0xA980, 0xA9CE, 0xA9CF, + 0xA9D0, 0xA9DA, 0xA9DE, 0xA9E0, 0xA9FF, 0xAA00, 0xAA37, 0xAA40, 0xAA4E, 0xAA50, + 0xAA5A, 0xAA5C, 0xAA60, 0xAA80, 0xAAC3, 0xAADB, 0xAAE0, 0xAAF7, 0xAB01, 0xAB07, + 0xAB09, 0xAB0F, 0xAB11, 0xAB17, 0xAB20, 0xAB27, 0xAB28, 0xAB2F, 0xAB30, 0xAB5B, + 0xAB5C, 0xAB65, 0xAB66, 0xAB6A, 0xAB6C, 0xAB70, 0xABC0, 0xABEE, 0xABF0, 0xABFA, + 0xAC00, 0xD7A4, 0xD7B0, 0xD7C7, 0xD7CB, 0xD7FC, 0xF900, 0xFA6E, 0xFA70, 0xFADA, + 0xFB00, 0xFB07, 0xFB13, 0xFB18, 0xFB1D, 0xFB37, 0xFB38, 0xFB3D, 0xFB3E, 0xFB3F, + 0xFB40, 0xFB42, 0xFB43, 0xFB45, 0xFB46, 0xFB50, 0xFBC3, 0xFBD3, 0xFD3E, 0xFD40, + 0xFD90, 0xFD92, 0xFDC8, 0xFDCF, 0xFDD0, 0xFDF0, 0xFE00, 0xFE10, 0xFE1A, 0xFE20, + 0xFE2E, 0xFE30, 0xFE53, 0xFE54, 0xFE67, 0xFE68, 0xFE6C, 0xFE70, 0xFE75, 0xFE76, + 0xFEFD, 0xFEFF, 0xFF00, 0xFF01, 0xFF21, 0xFF3B, 0xFF41, 0xFF5B, 0xFF66, 0xFF70, + 0xFF71, 0xFF9E, 0xFFA0, 0xFFBF, 0xFFC2, 0xFFC8, 0xFFCA, 0xFFD0, 0xFFD2, 0xFFD8, + 0xFFDA, 0xFFDD, 0xFFE0, 0xFFE7, 0xFFE8, 0xFFEF, 0xFFF9, 0xFFFE, 0x10000, 0x1000C, + 0x1000D, 0x10027, 0x10028, 0x1003B, 0x1003C, 0x1003E, 0x1003F, 0x1004E, 0x10050, 0x1005E, + 0x10080, 0x100FB, 0x10100, 0x10103, 0x10107, 0x10134, 0x10137, 0x10140, 0x1018F, 0x10190, + 0x1019D, 0x101A0, 0x101A1, 0x101D0, 0x101FD, 0x101FE, 0x10280, 0x1029D, 0x102A0, 0x102D1, + 0x102E0, 0x102E1, 0x102FC, 0x10300, 0x10324, 0x1032D, 0x10330, 0x1034B, 0x10350, 0x1037B, + 0x10380, 0x1039E, 0x1039F, 0x103A0, 0x103C4, 0x103C8, 0x103D6, 0x10400, 0x10450, 0x10480, + 0x1049E, 0x104A0, 0x104AA, 0x104B0, 0x104D4, 0x104D8, 0x104FC, 0x10500, 0x10528, 0x10530, + 0x10564, 0x1056F, 0x10570, 0x1057B, 0x1057C, 0x1058B, 0x1058C, 0x10593, 0x10594, 0x10596, + 0x10597, 0x105A2, 0x105A3, 0x105B2, 0x105B3, 0x105BA, 0x105BB, 0x105BD, 0x105C0, 0x105F4, + 0x10600, 0x10737, 0x10740, 0x10756, 0x10760, 0x10768, 0x10780, 0x10786, 0x10787, 0x107B1, + 0x107B2, 0x107BB, 0x10800, 0x10806, 0x10808, 0x10809, 0x1080A, 0x10836, 0x10837, 0x10839, + 0x1083C, 0x1083D, 0x1083F, 0x10840, 0x10856, 0x10857, 0x10860, 0x10880, 0x1089F, 0x108A7, + 0x108B0, 0x108E0, 0x108F3, 0x108F4, 0x108F6, 0x108FB, 0x10900, 0x1091C, 0x1091F, 0x10920, + 0x1093A, 0x1093F, 0x10940, 0x10980, 0x109A0, 0x109B8, 0x109BC, 0x109D0, 0x109D2, 0x10A00, + 0x10A04, 0x10A05, 0x10A07, 0x10A0C, 0x10A14, 0x10A15, 0x10A18, 0x10A19, 0x10A36, 0x10A38, + 0x10A3B, 0x10A3F, 0x10A49, 0x10A50, 0x10A59, 0x10A60, 0x10A80, 0x10AA0, 0x10AC0, 0x10AE7, + 0x10AEB, 0x10AF7, 0x10B00, 0x10B36, 0x10B39, 0x10B40, 0x10B56, 0x10B58, 0x10B60, 0x10B73, + 0x10B78, 0x10B80, 0x10B92, 0x10B99, 0x10B9D, 0x10BA9, 0x10BB0, 0x10C00, 0x10C49, 0x10C80, + 0x10CB3, 0x10CC0, 0x10CF3, 0x10CFA, 0x10D00, 0x10D28, 0x10D30, 0x10D3A, 0x10D40, 0x10D66, + 0x10D69, 0x10D86, 0x10D8E, 0x10D90, 0x10E60, 0x10E7F, 0x10E80, 0x10EAA, 0x10EAB, 0x10EAE, + 0x10EB0, 0x10EB2, 0x10EC2, 0x10EC5, 0x10EFC, 0x10F00, 0x10F28, 0x10F30, 0x10F5A, 0x10F70, + 0x10F8A, 0x10FB0, 0x10FCC, 0x10FE0, 0x10FF7, 0x11000, 0x1104E, 0x11052, 0x11076, 0x1107F, + 0x11080, 0x110C3, 0x110CD, 0x110CE, 0x110D0, 0x110E9, 0x110F0, 0x110FA, 0x11100, 0x11135, + 0x11136, 0x11148, 0x11150, 0x11177, 0x11180, 0x111E0, 0x111E1, 0x111F5, 0x11200, 0x11212, + 0x11213, 0x11242, 0x11280, 0x11287, 0x11288, 0x11289, 0x1128A, 0x1128E, 0x1128F, 0x1129E, + 0x1129F, 0x112AA, 0x112B0, 0x112EB, 0x112F0, 0x112FA, 0x11300, 0x11304, 0x11305, 0x1130D, + 0x1130F, 0x11311, 0x11313, 0x11329, 0x1132A, 0x11331, 0x11332, 0x11334, 0x11335, 0x1133A, + 0x1133B, 0x1133C, 0x11345, 0x11347, 0x11349, 0x1134B, 0x1134E, 0x11350, 0x11351, 0x11357, + 0x11358, 0x1135D, 0x11364, 0x11366, 0x1136D, 0x11370, 0x11375, 0x11380, 0x1138A, 0x1138B, + 0x1138C, 0x1138E, 0x1138F, 0x11390, 0x113B6, 0x113B7, 0x113C1, 0x113C2, 0x113C3, 0x113C5, + 0x113C6, 0x113C7, 0x113CB, 0x113CC, 0x113D6, 0x113D7, 0x113D9, 0x113E1, 0x113E3, 0x11400, + 0x1145C, 0x1145D, 0x11462, 0x11480, 0x114C8, 0x114D0, 0x114DA, 0x11580, 0x115B6, 0x115B8, + 0x115DE, 0x11600, 0x11645, 0x11650, 0x1165A, 0x11660, 0x1166D, 0x11680, 0x116BA, 0x116C0, + 0x116CA, 0x116D0, 0x116E4, 0x11700, 0x1171B, 0x1171D, 0x1172C, 0x11730, 0x11747, 0x11800, + 0x1183C, 0x118A0, 0x118F3, 0x118FF, 0x11900, 0x11907, 0x11909, 0x1190A, 0x1190C, 0x11914, + 0x11915, 0x11917, 0x11918, 0x11936, 0x11937, 0x11939, 0x1193B, 0x11947, 0x11950, 0x1195A, + 0x119A0, 0x119A8, 0x119AA, 0x119D8, 0x119DA, 0x119E5, 0x11A00, 0x11A48, 0x11A50, 0x11AA3, + 0x11AB0, 0x11AC0, 0x11AF9, 0x11B00, 0x11B0A, 0x11BC0, 0x11BE2, 0x11BF0, 0x11BFA, 0x11C00, + 0x11C09, 0x11C0A, 0x11C37, 0x11C38, 0x11C46, 0x11C50, 0x11C6D, 0x11C70, 0x11C90, 0x11C92, + 0x11CA8, 0x11CA9, 0x11CB7, 0x11D00, 0x11D07, 0x11D08, 0x11D0A, 0x11D0B, 0x11D37, 0x11D3A, + 0x11D3B, 0x11D3C, 0x11D3E, 0x11D3F, 0x11D48, 0x11D50, 0x11D5A, 0x11D60, 0x11D66, 0x11D67, + 0x11D69, 0x11D6A, 0x11D8F, 0x11D90, 0x11D92, 0x11D93, 0x11D99, 0x11DA0, 0x11DAA, 0x11EE0, + 0x11EF9, 0x11F00, 0x11F11, 0x11F12, 0x11F3B, 0x11F3E, 0x11F5B, 0x11FB0, 0x11FB1, 0x11FC0, + 0x11FF2, 0x11FFF, 0x12000, 0x1239A, 0x12400, 0x1246F, 0x12470, 0x12475, 0x12480, 0x12544, + 0x12F90, 0x12FF3, 0x13000, 0x13456, 0x13460, 0x143FB, 0x14400, 0x14647, 0x16100, 0x1613A, + 0x16800, 0x16A39, 0x16A40, 0x16A5F, 0x16A60, 0x16A6A, 0x16A6E, 0x16A70, 0x16ABF, 0x16AC0, + 0x16ACA, 0x16AD0, 0x16AEE, 0x16AF0, 0x16AF6, 0x16B00, 0x16B46, 0x16B50, 0x16B5A, 0x16B5B, + 0x16B62, 0x16B63, 0x16B78, 0x16B7D, 0x16B90, 0x16D40, 0x16D7A, 0x16E40, 0x16E9B, 0x16F00, + 0x16F4B, 0x16F4F, 0x16F88, 0x16F8F, 0x16FA0, 0x16FE0, 0x16FE1, 0x16FE2, 0x16FE4, 0x16FE5, + 0x16FF0, 0x16FF2, 0x17000, 0x187F8, 0x18800, 0x18B00, 0x18CD6, 0x18CFF, 0x18D00, 0x18D09, + 0x1AFF0, 0x1AFF4, 0x1AFF5, 0x1AFFC, 0x1AFFD, 0x1AFFF, 0x1B000, 0x1B001, 0x1B120, 0x1B123, + 0x1B132, 0x1B133, 0x1B150, 0x1B153, 0x1B155, 0x1B156, 0x1B164, 0x1B168, 0x1B170, 0x1B2FC, + 0x1BC00, 0x1BC6B, 0x1BC70, 0x1BC7D, 0x1BC80, 0x1BC89, 0x1BC90, 0x1BC9A, 0x1BC9C, 0x1BCA0, + 0x1BCA4, 0x1CC00, 0x1CCFA, 0x1CD00, 0x1CEB4, 0x1CF00, 0x1CF2E, 0x1CF30, 0x1CF47, 0x1CF50, + 0x1CFC4, 0x1D000, 0x1D0F6, 0x1D100, 0x1D127, 0x1D129, 0x1D167, 0x1D16A, 0x1D17B, 0x1D183, + 0x1D185, 0x1D18C, 0x1D1AA, 0x1D1AE, 0x1D1EB, 0x1D200, 0x1D246, 0x1D2C0, 0x1D2D4, 0x1D2E0, + 0x1D2F4, 0x1D300, 0x1D357, 0x1D360, 0x1D379, 0x1D400, 0x1D455, 0x1D456, 0x1D49D, 0x1D49E, + 0x1D4A0, 0x1D4A2, 0x1D4A3, 0x1D4A5, 0x1D4A7, 0x1D4A9, 0x1D4AD, 0x1D4AE, 0x1D4BA, 0x1D4BB, + 0x1D4BC, 0x1D4BD, 0x1D4C4, 0x1D4C5, 0x1D506, 0x1D507, 0x1D50B, 0x1D50D, 0x1D515, 0x1D516, + 0x1D51D, 0x1D51E, 0x1D53A, 0x1D53B, 0x1D53F, 0x1D540, 0x1D545, 0x1D546, 0x1D547, 0x1D54A, + 0x1D551, 0x1D552, 0x1D6A6, 0x1D6A8, 0x1D7CC, 0x1D7CE, 0x1D800, 0x1DA8C, 0x1DA9B, 0x1DAA0, + 0x1DAA1, 0x1DAB0, 0x1DF00, 0x1DF1F, 0x1DF25, 0x1DF2B, 0x1E000, 0x1E007, 0x1E008, 0x1E019, + 0x1E01B, 0x1E022, 0x1E023, 0x1E025, 0x1E026, 0x1E02B, 0x1E030, 0x1E06E, 0x1E08F, 0x1E090, + 0x1E100, 0x1E12D, 0x1E130, 0x1E13E, 0x1E140, 0x1E14A, 0x1E14E, 0x1E150, 0x1E290, 0x1E2AF, + 0x1E2C0, 0x1E2FA, 0x1E2FF, 0x1E300, 0x1E4D0, 0x1E4FA, 0x1E5D0, 0x1E5FB, 0x1E5FF, 0x1E600, + 0x1E7E0, 0x1E7E7, 0x1E7E8, 0x1E7EC, 0x1E7ED, 0x1E7EF, 0x1E7F0, 0x1E7FF, 0x1E800, 0x1E8C5, + 0x1E8C7, 0x1E8D7, 0x1E900, 0x1E94C, 0x1E950, 0x1E95A, 0x1E95E, 0x1E960, 0x1EC71, 0x1ECB5, + 0x1ED01, 0x1ED3E, 0x1EE00, 0x1EE04, 0x1EE05, 0x1EE20, 0x1EE21, 0x1EE23, 0x1EE24, 0x1EE25, + 0x1EE27, 0x1EE28, 0x1EE29, 0x1EE33, 0x1EE34, 0x1EE38, 0x1EE39, 0x1EE3A, 0x1EE3B, 0x1EE3C, + 0x1EE42, 0x1EE43, 0x1EE47, 0x1EE48, 0x1EE49, 0x1EE4A, 0x1EE4B, 0x1EE4C, 0x1EE4D, 0x1EE50, + 0x1EE51, 0x1EE53, 0x1EE54, 0x1EE55, 0x1EE57, 0x1EE58, 0x1EE59, 0x1EE5A, 0x1EE5B, 0x1EE5C, + 0x1EE5D, 0x1EE5E, 0x1EE5F, 0x1EE60, 0x1EE61, 0x1EE63, 0x1EE64, 0x1EE65, 0x1EE67, 0x1EE6B, + 0x1EE6C, 0x1EE73, 0x1EE74, 0x1EE78, 0x1EE79, 0x1EE7D, 0x1EE7E, 0x1EE7F, 0x1EE80, 0x1EE8A, + 0x1EE8B, 0x1EE9C, 0x1EEA1, 0x1EEA4, 0x1EEA5, 0x1EEAA, 0x1EEAB, 0x1EEBC, 0x1EEF0, 0x1EEF2, + 0x1F000, 0x1F02C, 0x1F030, 0x1F094, 0x1F0A0, 0x1F0AF, 0x1F0B1, 0x1F0C0, 0x1F0C1, 0x1F0D0, + 0x1F0D1, 0x1F0F6, 0x1F100, 0x1F1AE, 0x1F1E6, 0x1F200, 0x1F201, 0x1F203, 0x1F210, 0x1F23C, + 0x1F240, 0x1F249, 0x1F250, 0x1F252, 0x1F260, 0x1F266, 0x1F300, 0x1F6D8, 0x1F6DC, 0x1F6ED, + 0x1F6F0, 0x1F6FD, 0x1F700, 0x1F777, 0x1F77B, 0x1F7DA, 0x1F7E0, 0x1F7EC, 0x1F7F0, 0x1F7F1, + 0x1F800, 0x1F80C, 0x1F810, 0x1F848, 0x1F850, 0x1F85A, 0x1F860, 0x1F888, 0x1F890, 0x1F8AE, + 0x1F8B0, 0x1F8BC, 0x1F8C0, 0x1F8C2, 0x1F900, 0x1FA54, 0x1FA60, 0x1FA6E, 0x1FA70, 0x1FA7D, + 0x1FA80, 0x1FA8A, 0x1FA8F, 0x1FAC7, 0x1FACE, 0x1FADD, 0x1FADF, 0x1FAEA, 0x1FAF0, 0x1FAF9, + 0x1FB00, 0x1FB93, 0x1FB94, 0x1FBFA, 0x20000, 0x2A6E0, 0x2A700, 0x2B73A, 0x2B740, 0x2B81E, + 0x2B820, 0x2CEA2, 0x2CEB0, 0x2EBE1, 0x2EBF0, 0x2EE5E, 0x2F800, 0x2FA1E, 0x30000, 0x3134B, + 0x31350, 0x323B0, 0xE0001, 0xE0002, 0xE0020, 0xE0080, 0xE0100, 0xE01F0, + ]; + + /// The value of each range in . + private static ReadOnlySpan ScriptValues => + [ + 1, 76, 1, 76, 1, 76, 1, 76, 1, 76, 1, 76, 1, 76, 1, 76, 1, 16, 1, 2, 47, 1, 47, 0, + 47, 1, 47, 0, 47, 1, 47, 1, 47, 0, 47, 0, 47, 0, 47, 27, 47, 30, 2, 30, 0, 8, 0, 8, + 0, 8, 0, 55, 0, 55, 0, 55, 0, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 2, 6, 2, 6, + 1, 6, 143, 0, 143, 0, 143, 6, 154, 0, 105, 0, 105, 127, 0, 127, 0, 86, 0, 86, 0, 143, 0, 6, + 0, 6, 0, 6, 1, 6, 31, 2, 31, 1, 31, 14, 0, 14, 0, 14, 0, 14, 0, 14, 0, 14, 0, 14, + 0, 14, 0, 14, 0, 14, 0, 14, 0, 14, 0, 14, 0, 14, 0, 50, 0, 50, 0, 50, 0, 50, 0, 50, + 0, 50, 0, 50, 0, 50, 0, 50, 0, 50, 0, 50, 0, 50, 0, 50, 0, 50, 0, 50, 0, 50, 0, 48, + 0, 48, 0, 48, 0, 48, 0, 48, 0, 48, 0, 48, 0, 48, 0, 48, 0, 48, 0, 48, 0, 48, 0, 48, + 0, 48, 0, 111, 0, 111, 0, 111, 0, 111, 0, 111, 0, 111, 0, 111, 0, 111, 0, 111, 0, 111, 0, 111, + 0, 111, 0, 111, 0, 111, 0, 148, 0, 148, 0, 148, 0, 148, 0, 148, 0, 148, 0, 148, 0, 148, 0, 148, + 0, 148, 0, 148, 0, 148, 0, 148, 0, 148, 0, 148, 0, 148, 0, 151, 0, 151, 0, 151, 0, 151, 0, 151, + 0, 151, 0, 151, 0, 151, 0, 151, 0, 151, 0, 151, 0, 151, 0, 151, 71, 0, 71, 0, 71, 0, 71, 0, + 71, 0, 71, 0, 71, 0, 71, 0, 71, 0, 71, 0, 71, 0, 71, 0, 71, 0, 93, 0, 93, 0, 93, 0, + 93, 0, 93, 0, 93, 0, 93, 0, 135, 0, 135, 0, 135, 0, 135, 0, 135, 0, 135, 0, 135, 0, 135, 0, + 135, 0, 135, 0, 135, 0, 135, 0, 155, 0, 1, 155, 0, 75, 0, 75, 0, 75, 0, 75, 0, 75, 0, 75, + 0, 75, 0, 75, 0, 75, 0, 75, 0, 75, 0, 156, 0, 156, 0, 156, 0, 156, 0, 156, 0, 156, 1, 156, + 0, 99, 41, 0, 41, 0, 41, 0, 41, 1, 41, 51, 39, 0, 39, 0, 39, 0, 39, 0, 39, 0, 39, 0, + 39, 0, 39, 0, 39, 0, 39, 0, 39, 0, 39, 0, 39, 0, 39, 0, 39, 0, 39, 0, 39, 0, 39, 0, + 25, 0, 25, 0, 22, 107, 0, 126, 1, 126, 0, 153, 0, 153, 53, 1, 0, 20, 0, 144, 0, 144, 0, 144, + 0, 68, 0, 68, 0, 68, 0, 95, 1, 95, 1, 95, 0, 95, 0, 95, 0, 22, 0, 78, 0, 78, 0, 78, + 0, 78, 0, 78, 146, 0, 146, 0, 147, 0, 147, 0, 147, 0, 147, 68, 19, 0, 19, 74, 0, 74, 0, 74, + 0, 74, 0, 74, 0, 2, 0, 10, 0, 10, 140, 13, 0, 13, 77, 0, 77, 0, 77, 108, 30, 0, 41, 0, + 41, 140, 0, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 0, 76, 47, 30, 76, 47, 76, 47, 76, + 30, 76, 47, 2, 76, 47, 0, 47, 0, 47, 0, 47, 0, 47, 0, 47, 0, 47, 0, 47, 0, 47, 0, 47, + 0, 47, 0, 47, 0, 47, 0, 47, 0, 47, 0, 47, 0, 1, 2, 1, 0, 1, 76, 0, 1, 76, 1, 0, + 76, 0, 1, 0, 2, 0, 1, 47, 1, 76, 1, 76, 1, 76, 1, 76, 1, 0, 1, 0, 1, 0, 1, 18, + 1, 0, 1, 0, 1, 42, 76, 27, 0, 27, 41, 0, 41, 0, 41, 0, 152, 0, 152, 0, 152, 39, 0, 39, + 0, 39, 0, 39, 0, 39, 0, 39, 0, 39, 0, 39, 0, 39, 0, 30, 1, 0, 52, 0, 52, 0, 52, 0, + 1, 52, 1, 52, 1, 52, 2, 51, 1, 52, 1, 0, 56, 0, 2, 1, 56, 1, 65, 1, 65, 0, 16, 0, + 51, 0, 1, 16, 1, 0, 1, 65, 51, 0, 1, 51, 1, 65, 1, 65, 1, 52, 1, 52, 170, 0, 170, 0, + 81, 163, 0, 30, 11, 0, 1, 76, 1, 76, 0, 76, 0, 76, 0, 76, 0, 76, 142, 0, 1, 0, 118, 0, + 129, 0, 129, 0, 31, 64, 1, 64, 124, 0, 124, 51, 0, 63, 0, 1, 63, 0, 63, 99, 0, 24, 0, 24, + 0, 24, 0, 24, 99, 150, 0, 150, 97, 0, 39, 0, 39, 0, 39, 0, 39, 0, 39, 0, 76, 1, 76, 47, + 76, 1, 0, 25, 97, 0, 97, 0, 51, 0, 51, 0, 51, 0, 52, 0, 52, 0, 76, 0, 8, 0, 55, 0, + 55, 0, 55, 0, 55, 0, 55, 0, 55, 6, 0, 6, 1, 6, 0, 6, 0, 6, 0, 6, 2, 1, 0, 2, + 30, 1, 0, 1, 0, 1, 0, 6, 0, 6, 0, 1, 0, 1, 76, 1, 76, 1, 65, 1, 65, 1, 51, 0, + 51, 0, 51, 0, 51, 0, 51, 0, 1, 0, 1, 0, 1, 0, 80, 0, 80, 0, 80, 0, 80, 0, 80, 0, + 80, 0, 80, 0, 1, 0, 1, 0, 1, 47, 0, 1, 0, 47, 0, 1, 2, 0, 82, 0, 23, 0, 2, 1, + 0, 62, 0, 62, 45, 0, 117, 0, 162, 0, 162, 167, 0, 167, 0, 34, 131, 113, 0, 113, 0, 112, 0, 112, + 0, 37, 0, 4, 0, 4, 164, 0, 164, 0, 164, 0, 164, 0, 164, 0, 164, 0, 164, 0, 164, 0, 159, 0, + 79, 0, 79, 0, 79, 0, 76, 0, 76, 0, 76, 0, 29, 0, 29, 0, 29, 0, 29, 0, 29, 0, 29, 7, + 0, 7, 115, 103, 0, 103, 0, 54, 0, 54, 0, 54, 121, 0, 121, 83, 0, 83, 0, 92, 91, 0, 91, 0, + 91, 67, 0, 67, 0, 67, 0, 67, 0, 67, 0, 67, 0, 67, 0, 67, 0, 128, 102, 0, 87, 0, 87, 0, + 9, 0, 9, 123, 0, 123, 119, 0, 119, 120, 0, 120, 0, 120, 0, 110, 0, 61, 0, 61, 0, 61, 125, 0, + 125, 0, 40, 0, 40, 0, 40, 0, 6, 0, 169, 0, 169, 0, 169, 0, 6, 0, 6, 137, 0, 136, 0, 114, + 0, 26, 0, 38, 0, 17, 0, 17, 0, 17, 73, 0, 73, 0, 138, 0, 138, 0, 21, 0, 21, 0, 84, 0, + 132, 0, 135, 0, 69, 0, 69, 0, 98, 0, 98, 0, 98, 0, 98, 0, 98, 0, 134, 0, 134, 0, 46, 0, + 46, 0, 46, 0, 46, 0, 46, 0, 46, 0, 46, 0, 2, 46, 0, 46, 0, 46, 0, 46, 0, 46, 0, 46, + 0, 46, 0, 46, 0, 161, 0, 161, 0, 161, 0, 161, 0, 161, 0, 161, 0, 161, 0, 161, 0, 161, 0, 161, + 0, 161, 0, 104, 0, 104, 0, 157, 0, 157, 0, 133, 0, 133, 0, 94, 0, 94, 0, 95, 0, 145, 0, 145, + 0, 99, 0, 5, 0, 5, 0, 5, 0, 33, 0, 165, 0, 165, 32, 0, 32, 0, 32, 0, 32, 0, 32, 0, + 32, 0, 32, 0, 32, 0, 101, 0, 101, 0, 101, 0, 171, 0, 139, 0, 22, 116, 0, 31, 0, 141, 0, 141, + 0, 15, 0, 15, 0, 15, 0, 15, 0, 88, 0, 88, 0, 88, 0, 44, 0, 44, 0, 44, 0, 44, 0, 44, + 0, 44, 0, 44, 0, 43, 0, 43, 0, 43, 0, 43, 0, 43, 0, 43, 0, 85, 0, 66, 0, 66, 0, 66, + 0, 81, 0, 148, 0, 148, 168, 0, 168, 0, 168, 0, 168, 0, 28, 0, 36, 0, 36, 0, 57, 0, 49, 0, + 11, 0, 96, 0, 96, 0, 96, 158, 0, 158, 0, 12, 0, 12, 0, 58, 0, 58, 0, 58, 0, 58, 0, 58, + 0, 72, 0, 89, 0, 122, 0, 122, 0, 122, 0, 149, 106, 52, 70, 0, 52, 0, 149, 0, 149, 70, 0, 70, + 149, 0, 65, 0, 65, 0, 65, 0, 65, 56, 65, 0, 56, 0, 56, 0, 65, 0, 65, 0, 106, 0, 35, 0, + 35, 0, 35, 0, 35, 0, 35, 1, 0, 1, 0, 1, 0, 2, 0, 2, 0, 1, 0, 1, 0, 1, 0, 1, + 2, 1, 2, 1, 2, 1, 2, 1, 0, 47, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, + 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, + 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 130, 0, 130, 0, 130, 0, 76, 0, 76, 0, 42, 0, + 42, 0, 42, 0, 42, 0, 42, 0, 30, 0, 30, 0, 59, 0, 59, 0, 59, 0, 59, 0, 160, 0, 166, 0, + 166, 0, 100, 0, 109, 0, 109, 0, 39, 0, 39, 0, 39, 0, 39, 0, 90, 0, 90, 0, 3, 0, 3, 0, + 3, 0, 1, 0, 1, 0, 6, 0, 6, 0, 6, 0, 6, 0, 6, 0, 6, 0, 6, 0, 6, 0, 6, 0, + 6, 0, 6, 0, 6, 0, 6, 0, 6, 0, 6, 0, 6, 0, 6, 0, 6, 0, 6, 0, 6, 0, 6, 0, + 6, 0, 6, 0, 6, 0, 6, 0, 6, 0, 6, 0, 6, 0, 6, 0, 6, 0, 6, 0, 6, 0, 6, 0, + 6, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 56, 1, 0, 1, 0, 1, 0, + 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, + 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, + 1, 0, 1, 0, 52, 0, 52, 0, 52, 0, 52, 0, 52, 0, 52, 0, 52, 0, 52, 0, 52, 0, 1, 0, + 1, 0, 2, 0, + ]; +} diff --git a/src/Filee.Engines/Typesetting/UnicodeData.cs b/src/Filee.Engines/Typesetting/UnicodeData.cs new file mode 100644 index 00000000..16400638 --- /dev/null +++ b/src/Filee.Engines/Typesetting/UnicodeData.cs @@ -0,0 +1,64 @@ +// Unicode character properties the layout engine needs and .NET doesn't expose: Line_Break for line breaking +// (UAX #14), Bidi_Class for the bidirectional algorithm (UAX #9), Script for itemisation and shaping (UAX #24) and +// East_Asian_Width for CJK widths (UAX #11). The tables are generated from the Unicode Character Database by +// build/generate-unicode-tables.ps1 into UnicodeData.Generated.cs, at the Unicode version .NET uses, so these values +// and .NET's own (categories, grapheme clusters) describe the same characters. + +using System.Text; + +namespace Filee.Engines.Typesetting; + +/// Unicode property lookups for any code point (unassigned ones get the UCD's defaults). +internal static partial class UnicodeData +{ + /// The Line_Break class of , before UAX #14's tailoring (rule LB1). + public static LineBreakClass GetLineBreak(int codePoint) => (LineBreakClass)Lookup(LineBreakStarts, LineBreakValues, codePoint); + + /// The Bidi_Class of . + public static BidiClass GetBidiClass(int codePoint) => (BidiClass)Lookup(BidiStarts, BidiValues, codePoint); + + /// The East_Asian_Width of . + public static EastAsianWidth GetEastAsianWidth(int codePoint) => (EastAsianWidth)Lookup(EastAsianWidthStarts, EastAsianWidthValues, codePoint); + + /// The Script of (Common and Inherited included, not resolved). + public static UnicodeScript GetScript(int codePoint) => (UnicodeScript)Lookup(ScriptStarts, ScriptValues, codePoint); + + /// + public static LineBreakClass GetLineBreak(Rune rune) => GetLineBreak(rune.Value); + + /// + public static BidiClass GetBidiClass(Rune rune) => GetBidiClass(rune.Value); + + /// + public static EastAsianWidth GetEastAsianWidth(Rune rune) => GetEastAsianWidth(rune.Value); + + /// + public static UnicodeScript GetScript(Rune rune) => GetScript(rune.Value); + + /// The ISO 15924 code of a script ("Latn", "Hang", "Zyyy" for Common), as HarfBuzz names it. + public static string IsoCode(this UnicodeScript script) => ScriptCodes[(int)script]; + + /// True for East Asian Wide and Fullwidth characters: one em wide in CJK typography. + public static bool IsWide(int codePoint) => GetEastAsianWidth(codePoint) is EastAsianWidth.Wide or EastAsianWidth.Fullwidth; + + /// True for a strong right-to-left character (Bidi_Class R or AL). + public static bool IsRightToLeft(int codePoint) => GetBidiClass(codePoint) is BidiClass.R or BidiClass.AL; + + /// + /// The code point at : a surrogate pair as one, a lone surrogate as itself (it has its + /// own properties: Line_Break SG). + /// + public static int CodePointAt(ReadOnlySpan text, int index) => + Rune.DecodeFromUtf16(text[index..], out var rune, out _) == System.Buffers.OperationStatus.Done ? rune.Value : text[index]; + + /// The value of the range that holds ; outside the code space 0. + private static byte Lookup(ReadOnlySpan starts, ReadOnlySpan values, int codePoint) + { + if ((uint)codePoint > 0x10FFFF) + return 0; + // The last range that starts at or before the code point. BinarySearch gives the complement of the first larger + // start when there is no exact match. + var index = starts.BinarySearch(codePoint); + return values[index >= 0 ? index : ~index - 1]; + } +} diff --git a/tests/Filee.Engines.Tests/FontResolverTests.cs b/tests/Filee.Engines.Tests/FontResolverTests.cs new file mode 100644 index 00000000..5c4dc8f0 --- /dev/null +++ b/tests/Filee.Engines.Tests/FontResolverTests.cs @@ -0,0 +1,219 @@ +// The layout engine's font resolver (Typesetting/FontResolver.cs) against a fake list of installed fonts, like +// RhwpFontsTests: other names (함초롬바탕 → HCR Batang), style matching, variable and CFF2 fonts, replacements for +// missing fonts and per-cluster fallback. One test looks at the real installed fonts and skips without Malgun Gothic. + +using Filee.Engines.Typesetting; + +namespace Filee.Engines.Tests; + +public class FontResolverTests +{ + private const string Latin = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789 .,;:!?'\"()-“”‘’…"; + private const string Hangul = "한글문서대한민국섞인장입니다"; + private const string Han = "改造我们的学习这是一个测试文档说话国家和会时间漢字辻"; + private const string Kana = "日本語のテキストあいうえおアイウエオ"; + private const string Cyrillic = "АБВГДЕЖЗИКЛМНОПРСТУФХЦЧШЩЭЮЯабвгдежзиклмнопрстуфхцчшщэюя"; + private const string Arabic = "ابتثجحخدذرزسشصضطظعغفقكلمنهوي،"; + + /// A fake computer: families by their English names only, so Korean names must be mapped. + private static FakeFonts Computer() => new( + Face("Malgun Gothic", Latin + Hangul + "漢字國"), + Face("Malgun Gothic", Latin + Hangul + "漢字國", weight: 700), + Face("HCR Batang", Latin + Hangul + "漢字國"), + Face("Batang", Latin + Hangul + "漢字國"), + Face("HYSinMyeongJo-Medium", Latin + Hangul), + Face("Microsoft YaHei", Latin + Cyrillic + Han), + Face("SimSun", Latin + Han), + Face("Yu Gothic", Latin + Han + Kana), + Face("Segoe UI", Latin + Cyrillic + Arabic + "\u0301"), + Face("Times New Roman", Latin + Cyrillic + Arabic), + Face("Liberation Sans", Latin), + Face("Consolas", Latin), + Face("思源宋体 VF", Latin + Han, cff2: true, variable: true), + Face("Noto Sans KR", Latin + Hangul, variable: true), + Face("Pretendard", Latin + Hangul, weight: 400), + Face("Pretendard", Latin + Hangul, weight: 500), + Face("Pretendard", Latin + Hangul, weight: 700), + Face("Pretendard", Latin + Hangul, weight: 300, italic: true)); + + private static FontFace Face(string family, string chars, int weight = 400, bool italic = false, bool cff2 = false, bool variable = false) => + new(family, weight, italic, c => c <= 0xFFFF && chars.Contains((char)c)) { Cff2 = cff2, Variable = variable }; + + private static string? Family(ResolvedFont? font) => font?.Family; + + [Theory] + [InlineData("함초롬바탕", "HCR Batang")] + [InlineData("맑은 고딕", "Malgun Gothic")] + [InlineData("바탕", "Batang")] + [InlineData("한양신명조", "HYSinMyeongJo-Medium")] // 한글's built-in font → the TrueType font of the same design + [InlineData("HY신명조", "HYSinMyeongJo-Medium")] + [InlineData("Arial", "Liberation Sans")] // metric-compatible + [InlineData("malgun gothic", "Malgun Gothic")] + [InlineData("Not Installed", null)] + public void Fonts_are_found_by_their_other_names(string name, string? family) + { + Assert.Equal(family, Family(new FontResolver(Computer()).Find(name, FontStyle.Regular))); + } + + [Fact] + public void The_nearest_style_is_chosen_and_what_is_missing_is_synthesized() + { + var resolver = new FontResolver(Computer()); + + var bold = resolver.Find("맑은 고딕", FontStyle.From(bold: true, italic: false))!; + Assert.Equal(700, bold.Face.Weight); + Assert.False(bold.SyntheticBold); + + var batang = resolver.Find("바탕", FontStyle.From(bold: true, italic: true))!; + Assert.Equal(400, batang.Face.Weight); + Assert.True(batang.SyntheticBold); + Assert.True(batang.SyntheticItalic); + + // CSS weight matching: 400 prefers up to 500 first; 600 prefers heavier; 200 takes the lightest above it. + FontFace[] faces = [Face("X", "", 300), Face("X", "", 500), Face("X", "", 700)]; + Assert.Equal(500, FontResolver.Match(faces, new FontStyle(400)).Face.Weight); + Assert.Equal(700, FontResolver.Match(faces, new FontStyle(600)).Face.Weight); + Assert.Equal(300, FontResolver.Match(faces, new FontStyle(350)).Face.Weight); + Assert.Equal(300, FontResolver.Match(faces, new FontStyle(200)).Face.Weight); + Assert.Equal(500, FontResolver.Match([Face("X", "", 300), Face("X", "", 500)], new FontStyle(800)).Face.Weight); + } + + [Fact] + public void Style_words_in_a_name_set_the_weight() + { + var resolver = new FontResolver(Computer()); + + Assert.Equal(500, resolver.Find("Pretendard Medium", FontStyle.Regular)!.Face.Weight); + Assert.Equal(700, resolver.Find("Pretendard Medium", FontStyle.From(bold: true, italic: false))!.Face.Weight); + // Italic is matched before weight: the only italic face, though lighter. + var italic = resolver.Find("Pretendard", FontStyle.From(bold: false, italic: true))!; + Assert.Equal(300, italic.Face.Weight); + Assert.False(italic.SyntheticItalic); + } + + [Fact] + public void A_variable_font_is_its_default_instance() + { + var bold = new FontResolver(Computer()).Find("Noto Sans KR", FontStyle.From(bold: true, italic: false))!; + + Assert.True(bold.Face.Variable); + Assert.True(bold.SyntheticBold); // no instancer: the default instance, emboldened + } + + [Fact] + public void A_cff2_font_is_used_only_without_another_font_for_its_script() + { + Assert.Equal("SimSun", Family(new FontResolver(Computer()).Resolve("思源宋体 VF Medium", FontStyle.Regular))); + + var only = new FontResolver(new FakeFonts(Face("思源宋体 VF", Latin + Han, cff2: true))); + Assert.Equal("思源宋体 VF", Family(only.Resolve("思源宋体 VF Medium", FontStyle.Regular))); + } + + [Theory] + [InlineData("함초롬돋움", null, "Malgun Gothic")] // missing Korean sans → the first installed Korean sans + [InlineData("휴먼명조", null, "Batang")] // missing Korean serif (명조) → Korean serif + [InlineData("Unknown Sans", "ja", "Yu Gothic")] + [InlineData("Unknown Serif", "zh-Hans", "SimSun")] + [InlineData("Unknown", "", "Segoe UI")] + [InlineData("serif", "zh-Hans", "SimSun")] + [InlineData("sans-serif", "ko", "Malgun Gothic")] + [InlineData("monospace", "", "Consolas")] + [InlineData(null, "ja", "Yu Gothic")] + public void Missing_and_generic_fonts_get_one_for_the_script_and_style(string? name, string? language, string family) + { + Assert.Equal(family, Family(new FontResolver(Computer()).Resolve(name, FontStyle.Regular, language))); + } + + [Fact] + public void The_document_plan_follows_the_rhwp_rules() + { + // Issue #64: Korean default fonts in a Chinese document. + var plan = new FontResolver(Computer()).Plan([("HANJA", "맑은 고딕"), ("HANJA", "함초롬바탕")], "改造我们的学习 这是一个测试文档。"); + + Assert.Equal("Microsoft YaHei", plan[("HANJA", "맑은 고딕")]); + Assert.Equal("SimSun", plan[("HANJA", "함초롬바탕")]); + } + + [Fact] + public void Clusters_the_font_lacks_go_to_a_font_for_their_script() + { + var resolver = new FontResolver(Computer()); + var primary = resolver.Find("맑은 고딕", FontStyle.Regular)!; + const string text = "한글 مرحبا، abc"; + + var runs = resolver.Fallback(text, 0, text.Length, primary, FontStyle.Regular); + + // The Arabic word goes to Segoe UI with the punctuation and space after it; the Latin returns to the primary. + Assert.Equal([("한글 ", "Malgun Gothic"), ("مرحبا، ", "Segoe UI"), ("abc", "Malgun Gothic")], Describe(text, runs)); + } + + [Fact] + public void A_mark_stays_with_its_base_in_a_font_that_has_both() + { + var resolver = new FontResolver(Computer()); + var primary = resolver.Find("Malgun Gothic", FontStyle.Regular)!; + const string text = "cafe\u0301"; + + var runs = resolver.Fallback(text, 0, text.Length, primary, FontStyle.Regular); + + Assert.Equal([("caf", "Malgun Gothic"), ("e\u0301", "Segoe UI")], Describe(text, runs)); + } + + [Fact] + public void Han_characters_take_the_font_of_the_text_s_language() + { + var resolver = new FontResolver(Computer()); + var primary = resolver.Find("Malgun Gothic", FontStyle.Regular)!; + + const string japanese = "日本語のテキスト辻"; + Assert.All(resolver.Fallback(japanese, 0, japanese.Length, primary, FontStyle.Regular), r => Assert.Equal("Yu Gothic", r.Font.Family)); + const string chinese = "这是一个测试文档辻"; + Assert.All(resolver.Fallback(chinese, 0, chinese.Length, primary, FontStyle.Regular), r => Assert.Equal("Microsoft YaHei", r.Font.Family)); + // A part of the paragraph: the language still comes from the whole text. + Assert.Equal("Yu Gothic", Assert.Single(resolver.Fallback(japanese, 8, 1, primary, FontStyle.Regular)).Font.Family); + } + + [Fact] + public void The_platform_fallback_comes_last_and_the_primary_font_when_nothing_has_the_character() + { + var fonts = Computer(); + fonts.Fallbacks.Add(Face("Noto Sans Symbols 2", "♔")); + var resolver = new FontResolver(fonts); + var primary = resolver.Find("Malgun Gothic", FontStyle.Regular)!; + const string text = "a♔b\U0001F9FF"; + + var runs = resolver.Fallback(text, 0, text.Length, primary, FontStyle.Regular); + + Assert.Equal([("a", "Malgun Gothic"), ("♔", "Noto Sans Symbols 2"), ("b\U0001F9FF", "Malgun Gothic")], Describe(text, runs)); + } + + [Fact] + public void Installed_fonts_are_found_by_korean_and_english_names() + { + var resolver = new FontResolver(); + var malgun = resolver.Find("Malgun Gothic", FontStyle.Regular); + Assert.SkipUnless(malgun is not null, "Malgun Gothic is not installed"); + + var korean = resolver.Find("맑은 고딕", FontStyle.From(bold: true, italic: false))!; + Assert.True(korean.Face.Weight >= 600); + Assert.False(korean.SyntheticBold); + Assert.True(malgun.Face.Has('한')); + Assert.Equal(0, malgun.Font.FaceIndex); + Assert.True(malgun.Font.Data.Length > 1_000_000); + Assert.Null(resolver.Find("Filee No Such Family", FontStyle.Regular)); + } + + private static List<(string Text, string Family)> Describe(string text, List runs) => + runs.Select(r => (text.Substring(r.Start, r.Length), r.Font.Family)).ToList(); + + private sealed class FakeFonts(params FontFace[] faces) : IFontCollection + { + /// What the platform's own fallback answers with. + public List Fallbacks { get; } = []; + + public IReadOnlyList Family(string name) => + faces.Where(f => string.Equals(f.Family, name, StringComparison.OrdinalIgnoreCase)).ToList(); + + public FontFace? ForCharacter(int codePoint, FontStyle style, string? language) => Fallbacks.FirstOrDefault(f => f.Has(codePoint)); + } +} diff --git a/tests/Filee.Engines.Tests/FontSlotsTests.cs b/tests/Filee.Engines.Tests/FontSlotsTests.cs new file mode 100644 index 00000000..3e2b73f2 --- /dev/null +++ b/tests/Filee.Engines.Tests/FontSlotsTests.cs @@ -0,0 +1,97 @@ +// Character → font slot for 한글's seven language slots and Word's four run fonts (Typesetting/FontSlots.cs). + +using Filee.Engines.Hwp.Hwpx; +using Filee.Engines.Typesetting; + +namespace Filee.Engines.Tests; + +public class FontSlotsTests +{ + [Theory] + [InlineData('가', nameof(HLanguage.Hangul))] + [InlineData(0x1100, nameof(HLanguage.Hangul))] // conjoining jamo + [InlineData('ㄱ', nameof(HLanguage.Hangul))] // compatibility jamo + [InlineData('A', nameof(HLanguage.Latin))] + [InlineData('7', nameof(HLanguage.Latin))] + [InlineData(' ', nameof(HLanguage.Latin))] + [InlineData('.', nameof(HLanguage.Latin))] + [InlineData('é', nameof(HLanguage.Latin))] + [InlineData('×', nameof(HLanguage.Latin))] + [InlineData('漢', nameof(HLanguage.Hanja))] + [InlineData('あ', nameof(HLanguage.Japanese))] + [InlineData('ア', nameof(HLanguage.Japanese))] + [InlineData('ー', nameof(HLanguage.Japanese))] // Common, but Japanese + [InlineData('ｱ', nameof(HLanguage.Japanese))] + [InlineData('“', nameof(HLanguage.Symbol))] + [InlineData('…', nameof(HLanguage.Symbol))] + [InlineData('。', nameof(HLanguage.Symbol))] + [InlineData('→', nameof(HLanguage.Symbol))] + [InlineData('Ａ', nameof(HLanguage.Symbol))] // fullwidth form + [InlineData('α', nameof(HLanguage.Other))] + [InlineData('Ж', nameof(HLanguage.Other))] + [InlineData('ب', nameof(HLanguage.Other))] + [InlineData('ก', nameof(HLanguage.Other))] + [InlineData(0xE000, nameof(HLanguage.User))] + public void Hwp_language_slots(int codePoint, string expected) + { + Assert.Equal(Enum.Parse(expected), FontSlots.Hwp(codePoint)); + } + + [Theory] + [InlineData('A', nameof(WordFontHint.Default), nameof(WordFontSlot.Ascii))] + [InlineData('é', nameof(WordFontHint.Default), nameof(WordFontSlot.HighAnsi))] + [InlineData('Ж', nameof(WordFontHint.Default), nameof(WordFontSlot.HighAnsi))] + [InlineData('“', nameof(WordFontHint.Default), nameof(WordFontSlot.HighAnsi))] + [InlineData('“', nameof(WordFontHint.EastAsia), nameof(WordFontSlot.EastAsia))] // ambiguous width with the East Asian hint + [InlineData('Ж', nameof(WordFontHint.EastAsia), nameof(WordFontSlot.EastAsia))] + [InlineData('é', nameof(WordFontHint.EastAsia), nameof(WordFontSlot.EastAsia))] + [InlineData('A', nameof(WordFontHint.EastAsia), nameof(WordFontSlot.Ascii))] + [InlineData('가', nameof(WordFontHint.Default), nameof(WordFontSlot.EastAsia))] + [InlineData('漢', nameof(WordFontHint.Default), nameof(WordFontSlot.EastAsia))] + [InlineData('。', nameof(WordFontHint.Default), nameof(WordFontSlot.EastAsia))] + [InlineData('Ａ', nameof(WordFontHint.Default), nameof(WordFontSlot.EastAsia))] + [InlineData('ب', nameof(WordFontHint.Default), nameof(WordFontSlot.ComplexScript))] + [InlineData('א', nameof(WordFontHint.Default), nameof(WordFontSlot.ComplexScript))] + [InlineData(0x0915, nameof(WordFontHint.Default), nameof(WordFontSlot.ComplexScript))] + [InlineData('ก', nameof(WordFontHint.Default), nameof(WordFontSlot.ComplexScript))] + [InlineData(0xE000, nameof(WordFontHint.Default), nameof(WordFontSlot.HighAnsi))] + [InlineData(0xE000, nameof(WordFontHint.EastAsia), nameof(WordFontSlot.EastAsia))] + public void Word_run_fonts(int codePoint, string hint, string expected) + { + Assert.Equal(Enum.Parse(expected), FontSlots.Word(codePoint, Enum.Parse(hint))); + } + + [Fact] + public void Complex_script_runs_use_the_cs_font_for_everything() + { + Assert.Equal(WordFontSlot.ComplexScript, FontSlots.Word('A', complexScript: true)); + Assert.Equal("Arial", new HWordFonts("Calibri", "Calibri", "맑은 고딕", "Arial").Font(WordFontSlot.ComplexScript)); + Assert.Equal(WordFontHint.EastAsia, FontSlots.Hint("eastAsia")); + } + + [Fact] + public void Runs_split_by_slot_and_keep_clusters_whole() + { + // "e" + combining acute stays Latin; the jamo sequence is one cluster in the Hangul slot. + const string text = "Cafe\u0301 한글 \u1100\u1161\u11A8 “漢字”"; + + var runs = FontSlots.HwpRuns(text, 0, text.Length); + + Assert.Equal( + [("Cafe\u0301 ", HLanguage.Latin), ("한글", HLanguage.Hangul), (" ", HLanguage.Latin), ("\u1100\u1161\u11A8", HLanguage.Hangul), + (" ", HLanguage.Latin), ("“", HLanguage.Symbol), ("漢字", HLanguage.Hanja), ("”", HLanguage.Symbol)], + runs.Select(r => (text.Substring(r.Start, r.Length), r.Slot))); + } + + [Fact] + public void Word_runs_cover_a_part_of_the_text() + { + const string text = "xx Hello 世界 مرحبا xx"; + + var runs = FontSlots.WordRuns(text, 3, text.Length - 6); + + Assert.Equal( + [("Hello ", WordFontSlot.Ascii), ("世界", WordFontSlot.EastAsia), (" ", WordFontSlot.Ascii), ("مرحبا", WordFontSlot.ComplexScript)], + runs.Select(r => (text.Substring(r.Start, r.Length), r.Slot))); + } +} diff --git a/tests/Filee.Engines.Tests/PdfPageWriterTests.cs b/tests/Filee.Engines.Tests/PdfPageWriterTests.cs index 7bc23ffd..7b4737b6 100644 --- a/tests/Filee.Engines.Tests/PdfPageWriterTests.cs +++ b/tests/Filee.Engines.Tests/PdfPageWriterTests.cs @@ -1,18 +1,16 @@ // The Filee renderer's PDF output (Rendering/PdfPageWriter.cs): Skia's PDF backend with fonts subset by Filee. The // acceptance page of the "PDF backend and font subsetting" spike is built here in code: one A4 page with Korean, Latin, -// Arabic and Devanagari paragraphs shaped by HarfBuzz, a link, outline entries and some non-text drawing. It needs -// installed fonts (Malgun Gothic, Segoe UI, Nirmala UI on Windows) and is skipped where they are missing, like the other -// font-dependent tests. Set FILEE_RENDER_OUT to a folder to keep the PDFs for a look in other viewers. +// Arabic and Devanagari paragraphs shaped by the layout engine's shaper (Typesetting/Shaper.cs), a link, outline entries +// and some non-text drawing. It needs installed fonts (Malgun Gothic, Segoe UI, Nirmala UI on Windows) and is skipped +// where they are missing, like the other font-dependent tests. Set FILEE_RENDER_OUT to a folder to keep the PDFs for a +// look in other viewers. -using System.Runtime.InteropServices; using System.Text; using Filee.Engines.Rendering; -using HarfBuzzSharp; using PdfSharp.Pdf.IO; using PDFtoImage; using SkiaSharp; using UglyToad.PdfPig.Content; -using Buffer = HarfBuzzSharp.Buffer; using PdfPigDocument = UglyToad.PdfPig.PdfDocument; #pragma warning disable CA1416 @@ -257,64 +255,21 @@ private List Lines(PageFont font, float size, string body, float maxWidt return lines; } - /// HarfBuzz shaping of one run in one font (the layout engine's shaper is a later step). + /// Runs shaped by the layout engine's shaper, placed with the pen starting at x on baseline y. private sealed class Shaper : IDisposable { - private readonly Dictionary _fonts = new(ReferenceEqualityComparer.Instance); + private readonly Typesetting.Shaper _shaper = new(); - public float Width(PageFont font, float size, string text) => Shape(font, size, text, 0, 0).Width; + public float Width(PageFont font, float size, string text) => _shaper.Shape(font, text).Width(size); /// Shapes with the pen starting at x on baseline y (page coordinates). public (GlyphRun Run, float Width) Shape(PageFont font, float size, string text, float x, float y, SKColor? color = null) { - var (hb, upem) = For(font); - using var buffer = new Buffer(); - buffer.AddUtf16(text); - buffer.GuessSegmentProperties(); - hb.Shape(buffer); - var infos = buffer.GlyphInfos; - var positions = buffer.GlyphPositions; - var scale = size / upem; - var glyphs = new ushort[infos.Length]; - var points = new SKPoint[infos.Length]; - var clusters = new int[infos.Length]; - var pen = 0f; - for (var i = 0; i < infos.Length; i++) - { - glyphs[i] = (ushort)infos[i].Codepoint; - clusters[i] = (int)infos[i].Cluster; - points[i] = new SKPoint(x + pen + positions[i].XOffset * scale, y - positions[i].YOffset * scale); - pen += positions[i].XAdvance * scale; - } - return (new GlyphRun(font, size, glyphs, points, text, clusters) { Color = color ?? SKColors.Black }, pen); + var run = _shaper.Shape(font, text); + return (run.ToGlyphRun(font, size, new SKPoint(x, y), text) with { Color = color ?? SKColors.Black }, run.Width(size)); } - private (HarfBuzzSharp.Font Font, int Upem) For(PageFont font) - { - if (!_fonts.TryGetValue(font, out var entry)) - { - // A native copy the blob owns: HarfBuzz reads the font data for as long as the face lives. - var memory = Marshal.AllocHGlobal(font.Data.Length); - Marshal.Copy(font.Data, 0, memory, font.Data.Length); - var blob = new Blob(memory, font.Data.Length, MemoryMode.ReadOnly, () => Marshal.FreeHGlobal(memory)); - var face = new Face(blob, font.FaceIndex); - var hb = new HarfBuzzSharp.Font(face); - hb.SetScale(face.UnitsPerEm, face.UnitsPerEm); - hb.SetFunctionsOpenType(); - _fonts[font] = entry = (blob, face, hb); - } - return (entry.Font, entry.Face.UnitsPerEm); - } - - public void Dispose() - { - foreach (var (blob, face, font) in _fonts.Values) - { - font.Dispose(); - face.Dispose(); - blob.Dispose(); - } - } + public void Dispose() => _shaper.Dispose(); } // ───────────────────────── Checks ───────────────────────── diff --git a/tests/Filee.Engines.Tests/ShaperTests.cs b/tests/Filee.Engines.Tests/ShaperTests.cs new file mode 100644 index 00000000..5346f5de --- /dev/null +++ b/tests/Filee.Engines.Tests/ShaperTests.cs @@ -0,0 +1,368 @@ +// The layout engine's HarfBuzz shaper (Typesetting/Shaper.cs) on Arabic, Devanagari, Gujarati, Thai and Hangul jamo +// text written here. Each script is checked three ways: +// - against HarfBuzz itself: the same text shaped the plain way (one buffer, properties guessed by HarfBuzz) gives the +// same glyphs, clusters, advances and offsets; +// - against recorded HarfBuzz output (hb-shape's text format) for the pinned Noto fonts build/fetch-fonts.ps1 puts in +// src/Filee.App/Assets/Fonts (CI fetches them; without them those tests skip; there is no pinned Gujarati font); +// - by what the script needs, independent of the font: Arabic joining forms and lam-alef, Indic vowel signs before +// their consonant and conjuncts, Thai SARA AM split, Hangul jamo composed into the syllable, no dotted circles. +// Fonts come from the repository's Noto fonts or the installed ones (Segoe UI, Nirmala UI, Leelawadee UI, Malgun +// Gothic, ...); a script is skipped when no font for it is found. + +using System.Runtime.InteropServices; +using System.Text; +using Filee.Engines.Rendering; +using Filee.Engines.Typesetting; +using HarfBuzzSharp; +using SkiaSharp; +using Buffer = HarfBuzzSharp.Buffer; + +namespace Filee.Engines.Tests; + +public sealed class ShaperTests : IDisposable +{ + private const string Arabic = "السلام عليكم، لا بأس. بِسْمِ اللَّهِ"; + private const string Devanagari = "क्षत्रिय कि हिन्दी श्री"; + private const string Gujarati = "ક્ષત્રિય કિ ગુજરાતી શ્રી"; + private const string Thai = "ที่นี่ น้ำ กำลัง"; + private const string Jamo = "\u1100\u1161\u11A8 \u1112\u1161\u11AB\u1100\u1173\u11AF"; + + /// Recorded HarfBuzz 8.3 output (hb-shape's text format, font units) for the pinned Noto fonts. + private static readonly Dictionary Recorded = new() + { + ["NotoSansArabic-Regular.ttf"] = + "[427=34@111,0+0|85=34+486|371=31@-35,456+0|368=31@-27,256+0|70=31+300|72=30+260|8=29+238|3=28+260|" + + "427=26@139,0+0|77=26+562|379=24@301,-126+0|36=24+850|427=22@-35,-191+0|317=22@31,-3+0|19=22+269|" + + "3=21+260|172=20+268|34=19+1209|298=18@-17,126+0|13=18+289|317=17@31,-3+0|19=17+269|3=16+260|" + + "10=15+363|73=14+219|3=13+260|176=12+328|77=11+562|60=10+508|319=9@39,-5+0|18=9+373|70=8+300|" + + "49=7+503|3=6+260|74=5+484|11=4+379|71=3+220|36=2+850|72=1+260|8=0+238]", + ["NotoSansDevanagari-Regular.ttf"] = + "[90=0+717|545=3+259|304=3+552|81=7+580|3=8+260|545=9+259|56=9+768|3=11+260|544=12+259|88=12+531|" + + "245=14+309|73=16+531|33=16+259|3=18+260|317=19+708|33=19+259]", + ["NotoSansThai-Regular.ttf"] = + "[117=0+609|94=0@-3,0+0|44=0@-3,0+0|71=3+613|94=3+0|44=3+0|111=6+260|71=7+613|59=7+0|49=7@-29,0+0|" + + "86=7+406|111=10+260|29=11+600|59=11@-2,0+0|86=11+406|33=13+571|45=13@-7,0+0|58=15+537]", + ["NotoSansKR-Regular.otf"] = + "[10317=0+920|1=3+224|20904=4+920|10828=7+920]", + }; + + private readonly Shaper _shaper = new(); + private readonly List _fonts = []; + + public void Dispose() + { + _shaper.Dispose(); + foreach (var font in _fonts) + font.Dispose(); + } + + /// Each script's text with the pinned Noto font and the fonts systems install for it. + public static TheoryData Fonts() => new() + { + { "NotoSansArabic-Regular.ttf", Arabic }, { "segoeui.ttf", Arabic }, { "arial.ttf", Arabic }, { "DejaVuSans.ttf", Arabic }, + { "NotoSansDevanagari-Regular.ttf", Devanagari }, { "Nirmala.ttc", Devanagari }, { "Lohit-Devanagari.ttf", Devanagari }, + { "Nirmala.ttc", Gujarati }, { "shruti.ttf", Gujarati }, { "NotoSansGujarati-Regular.ttf", Gujarati }, { "Lohit-Gujarati.ttf", Gujarati }, + { "NotoSansThai-Regular.ttf", Thai }, { "LeelawUI.ttf", Thai }, { "tahoma.ttf", Thai }, + { "NotoSansKR-Regular.otf", Jamo }, { "malgun.ttf", Jamo }, { "NotoSansCJK-Regular.ttc", Jamo }, + }; + + // ───────────────────────── Against HarfBuzz ───────────────────────── + + [Theory] + [MemberData(nameof(Fonts))] + public void Shaping_matches_harfbuzz(string file, string text) + { + var path = FindFont(file); + Assert.SkipUnless(path is not null, $"{file} is not installed"); + var font = Load(path); + + var run = _shaper.Shape(font, text); + + Assert.Equal(Reference(font, text), Serialize(run)); + } + + [Theory] + [InlineData("NotoSansArabic-Regular.ttf", Arabic)] + [InlineData("NotoSansDevanagari-Regular.ttf", Devanagari)] + [InlineData("NotoSansThai-Regular.ttf", Thai)] + [InlineData("NotoSansKR-Regular.otf", Jamo)] + public void Shaping_matches_recorded_harfbuzz_output(string file, string text) + { + var path = RepositoryFont(file); + Assert.SkipUnless(path is not null, $"{file} is not in src/Filee.App/Assets/Fonts (build/fetch-fonts.ps1)"); + var font = Load(path); + + var run = _shaper.Shape(font, text); + + Assert.Equal(Recorded[file], Serialize(run)); + Assert.Equal(Recorded[file], Reference(font, text)); + } + + // ───────────────────────── What each script needs ───────────────────────── + + [Fact] + public void Arabic_joins_right_to_left_with_contextual_forms_and_lam_alef() + { + var font = FontFor("Arabic"); + Assert.SkipUnless(font is not null, "no Arabic font found"); + + // Some fonts draw the dots of a letter as glyphs of their own (without advance), so letters are compared by the + // glyph that carries the advance. + var word = _shaper.Shape(font, "ببب"); + Assert.Equal(TextDirection.RightToLeft, word.Direction); + Assert.Equal([2, 1, 0], word.Clusters.Distinct()); // visual order: the last letter first + Assert.Equal(word.Clusters.OrderDescending(), word.Clusters); + var forms = Enumerable.Range(0, 3).Select(c => Base(word, c)).ToList(); + Assert.Equal(3, forms.Distinct().Count()); // initial, medial and final forms + Assert.DoesNotContain(Base(_shaper.Shape(font, "ب"), 0), forms); // none is the isolated form + + // Lam before alef is the lam-alef ligature (or a form of its own), not the lam that starts other words. + Assert.NotEqual(Base(_shaper.Shape(font, "لب"), 0), Base(_shaper.Shape(font, "لا"), 0)); + + var marked = _shaper.Shape(font, "بِ"); // a kasra: zero advance, in its letter's cluster + Assert.All(marked.Clusters, c => Assert.Equal(0, c)); + Assert.Contains(0, marked.XAdvances); + NoMissingGlyphs(font, _shaper.Shape(font, Arabic)); + } + + [Theory] + [InlineData("Devanagari", "\u0915\u093F", "\u0915\u094D\u0937", '\u0915', '\u093F')] // कि, क्ष + [InlineData("Gujarati", "\u0A95\u0ABF", "\u0A95\u0ACD\u0AB7", '\u0A95', '\u0ABF')] // કિ, ક્ષ + public void Indic_vowel_signs_reorder_and_conjuncts_form(string script, string syllable, string conjunct, char consonant, char sign) + { + var font = FontFor(script); + Assert.SkipUnless(font is not null, $"no {script} font found"); + + // The i vowel sign is written after its consonant and drawn before it. + var reordered = _shaper.Shape(font, syllable); + Assert.Equal(2, reordered.Glyphs.Length); + Assert.Equal([0, 0], reordered.Clusters); + Assert.NotEqual(Nominal(font, consonant), reordered.Glyphs[0]); + Assert.Equal(Nominal(font, consonant), reordered.Glyphs[1]); + + // Consonant + virama + consonant is one conjunct cluster with fewer glyphs than characters. + var joined = _shaper.Shape(font, conjunct); + Assert.True(joined.Glyphs.Length < conjunct.Length); + Assert.All(joined.Clusters, c => Assert.Equal(0, c)); + NoMissingGlyphs(font, _shaper.Shape(font, script == "Devanagari" ? Devanagari : Gujarati)); + + // A vowel sign without a consonant gets a dotted circle, as HarfBuzz marks broken clusters. + Assert.Contains(Nominal(font, '\u25CC'), _shaper.Shape(font, sign.ToString()).Glyphs); + } + + [Fact] + public void Thai_sara_am_is_split_and_marks_stay_in_their_cluster() + { + var font = FontFor("Thai"); + Assert.SkipUnless(font is not null, "no Thai font found"); + + // น้ำ: NO NU, MAI THO, SARA AM. SARA AM becomes NIKHAHIT + SARA AA. + var run = _shaper.Shape(font, "น้ำ"); + Assert.Contains(Nominal(font, 'า'), run.Glyphs); + Assert.True(run.Glyphs.Length >= 4); + Assert.All(run.Clusters, c => Assert.Equal(0, c)); + + // ที่: the tone mark sits on the vowel above, without an advance of its own. + var stacked = _shaper.Shape(font, "ที่"); + Assert.Equal(3, stacked.Glyphs.Length); + Assert.Equal(0, stacked.XAdvances[2]); + NoMissingGlyphs(font, _shaper.Shape(font, Thai)); + } + + [Fact] + public void Hangul_jamo_compose_into_syllables() + { + var font = FontFor("Hangul"); + Assert.SkipUnless(font is not null, "no Hangul font found"); + + var run = _shaper.Shape(font, Jamo); + + // ᄀ ᅡ ᆨ is 각, ᄒ ᅡ ᆫ 한, ᄀ ᅳ ᆯ 글: one glyph each, the precomposed syllable's. + Assert.Equal([Nominal(font, '각'), Nominal(font, ' '), Nominal(font, '한'), Nominal(font, '글')], run.Glyphs); + Assert.Equal([0, 3, 4, 7], run.Clusters); + + // Old Hangul (ᄒ ᆞ ᆫ) has no precomposed syllable: the jamo stay one cluster, drawn by the font's jamo features. + const string old = "\u1112\u119E\u11AB"; + if (font.Typeface.GetGlyph(0x119E) != 0) + { + var archaic = _shaper.Shape(font, old); + Assert.All(archaic.Clusters, c => Assert.Equal(0, c)); + NoMissingGlyphs(font, archaic); + } + } + + // ───────────────────────── Runs, context and cache ───────────────────────── + + [Fact] + public void A_run_is_shaped_with_the_text_around_it() + { + var font = FontFor("Arabic"); + Assert.SkipUnless(font is not null, "no Arabic font found"); + const string word = "بببب"; + + // Two runs of one word (a colour change in the middle) join like the whole word. + var whole = _shaper.Shape(font, word); + var first = _shaper.Shape(font, word, 0, 2); + var second = _shaper.Shape(font, word, 2, 2); + + Assert.Equal(whole.Glyphs, second.Glyphs.Concat(first.Glyphs)); // right to left: the second run is drawn first + Assert.Equal([1, 0], first.Clusters.Distinct()); + Assert.Equal([1, 0], second.Clusters.Distinct()); // counted from the run's start + } + + [Fact] + public void Runs_are_cached_by_text_context_and_options() + { + using var shaper = new Shaper(); + var font = TestFont(); + + var once = shaper.Shape(font, "Filee 한글"); + Assert.Same(once, shaper.Shape(font, "Filee 한글")); + Assert.Same(once, shaper.Shape(font, "Filee 한글", 0, 8)); + Assert.Equal(1, shaper.Shaped); + + shaper.Shape(font, "x Filee 한글", 2, 8); // other context + shaper.Shape(font, "Filee 한글", new ShapeOptions(Features: [new FontFeature("kern", 0)])); + shaper.Shape(font, "Filee 한글", new ShapeOptions(Language: "ko")); + Assert.Equal(4, shaper.Shaped); + } + + [Fact] + public void A_shaped_run_becomes_a_glyph_run_of_the_page_model() + { + var font = TestFont(); + var run = _shaper.Shape(font, "Filee 한글"); + + var glyphs = run.ToGlyphRun(font, 20, new SKPoint(10, 50), "Filee 한글", scaleX: 0.5f); + + glyphs.Validate(); + Assert.Equal("Filee 한글", glyphs.Text); + Assert.Equal(new SKPoint(10, 50), glyphs.Positions[0]); + Assert.Equal(10 + run.Width(20) * 0.5f, glyphs.Positions[^1].X + run.XAdvances[^1] * 20f / run.UnitsPerEm * 0.5f, 3); + Assert.Equal(0.5f, glyphs.ScaleX); + + var metrics = _shaper.Metrics(font); + Assert.Equal(run.UnitsPerEm, metrics.UnitsPerEm); + Assert.True(metrics.Ascender > 0 && metrics.Descender < 0 && metrics.UnderlineThickness > 0); + } + + // ───────────────────────── Fonts and the reference ───────────────────────── + + + /// The test font every system has (Latin and Korean, see FontTests). + private PageFont TestFont() => Load(Path.Combine(AppContext.BaseDirectory, "Fonts", "FileeTest-TrueType.ttf")); + + /// A font for a script: the repository's pinned Noto font, else an installed one, else null. + private PageFont? FontFor(string script) + { + string[] names = script switch + { + "Arabic" => ["NotoSansArabic-Regular.ttf", "segoeui.ttf", "arial.ttf", "tahoma.ttf", "DejaVuSans.ttf"], + "Devanagari" => ["NotoSansDevanagari-Regular.ttf", "Nirmala.ttc", "Nirmala.ttf", "mangal.ttf", "Lohit-Devanagari.ttf"], + "Gujarati" => ["NotoSansGujarati-Regular.ttf", "Nirmala.ttc", "Nirmala.ttf", "shruti.ttf", "Lohit-Gujarati.ttf"], + "Thai" => ["NotoSansThai-Regular.ttf", "LeelawUI.ttf", "tahoma.ttf"], + _ => ["NotoSansKR-Regular.otf", "malgun.ttf", "NotoSansCJK-Regular.ttc"], + }; + foreach (var name in names) + { + if (FindFont(name) is { } path) + return Load(path); + } + return null; + } + + private PageFont Load(string path) + { + var font = PageFont.FromFile(path); + _fonts.Add(font); + return font; + } + + /// A font build/fetch-fonts.ps1 put into src/Filee.App/Assets/Fonts; null when it isn't there. + private static string? RepositoryFont(string name) + { + for (var dir = new DirectoryInfo(AppContext.BaseDirectory); dir is not null; dir = dir.Parent) + { + if (File.Exists(Path.Combine(dir.FullName, "Filee.slnx"))) + { + var path = Path.Combine(dir.FullName, "src", "Filee.App", "Assets", "Fonts", name); + return File.Exists(path) ? path : null; + } + } + return null; + } + + /// A font file from the repository's app fonts, or the system's or the user's font folders. + private static string? FindFont(string name) + { + if (RepositoryFont(name) is { } pinned) + return pinned; + string[] folders = + [ + Environment.GetFolderPath(Environment.SpecialFolder.Fonts), + Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "Microsoft", "Windows", "Fonts"), + "/usr/share/fonts", "/usr/local/share/fonts", "/Library/Fonts", "/System/Library/Fonts", + ]; + foreach (var folder in folders.Where(f => f.Length > 0 && Directory.Exists(f))) + { + var match = Directory.EnumerateFiles(folder, name, new EnumerationOptions { RecurseSubdirectories = true, MatchCasing = MatchCasing.CaseInsensitive, IgnoreInaccessible = true }) + .FirstOrDefault(); + if (match is not null) + return match; + } + return null; + } + + private static ushort Nominal(PageFont font, int codePoint) => font.Typeface.GetGlyph(codePoint); + + /// The glyph of a cluster that carries its advance (the letter, not its dots or marks). + private static ushort Base(ShapedRun run, int cluster) => + Enumerable.Range(0, run.Glyphs.Length).Where(i => run.Clusters[i] == cluster).MaxBy(i => run.XAdvances[i]) is var i ? run.Glyphs[i] : (ushort)0; + + /// No .notdef and no dotted circle: every character was drawn and every cluster was well formed. + private static void NoMissingGlyphs(PageFont font, ShapedRun run) + { + Assert.DoesNotContain((ushort)0, run.Glyphs); + var circle = Nominal(font, '\u25CC'); + if (circle != 0) + Assert.DoesNotContain(circle, run.Glyphs); + } + + /// + /// The text shaped by HarfBuzz the plain way: one buffer with all the text, segment properties guessed by HarfBuzz, + /// the font at its units per em; serialized by HarfBuzz in hb-shape's text format. + /// + private static string Reference(PageFont font, string text) + { + var memory = Marshal.AllocHGlobal(font.Data.Length); + Marshal.Copy(font.Data, 0, memory, font.Data.Length); + using var blob = new Blob(memory, font.Data.Length, MemoryMode.ReadOnly, () => Marshal.FreeHGlobal(memory)); + using var face = new Face(blob, font.FaceIndex); + using var hb = new HarfBuzzSharp.Font(face); + hb.SetFunctionsOpenType(); + hb.SetScale(face.UnitsPerEm, face.UnitsPerEm); + using var buffer = new Buffer(); + buffer.AddUtf16(text); + buffer.GuessSegmentProperties(); + hb.Shape(buffer); + return buffer.SerializeGlyphs(hb, SerializeFormat.Text, SerializeFlag.NoGlyphNames); + } + + /// A shaped run in hb-shape's text format: [glyph=cluster@x,y+advance|...], offsets when not 0. + private static string Serialize(ShapedRun run) + { + var text = new StringBuilder("["); + for (var i = 0; i < run.Glyphs.Length; i++) + { + if (i > 0) + text.Append('|'); + text.Append(run.Glyphs[i]).Append('=').Append(run.Clusters[i]); + if (run.XOffsets[i] != 0 || run.YOffsets[i] != 0) + text.Append('@').Append(run.XOffsets[i]).Append(',').Append(run.YOffsets[i]); + text.Append('+').Append(run.XAdvances[i]); + if (run.YAdvances[i] != 0) + text.Append(',').Append(run.YAdvances[i]); + } + return text.Append(']').ToString(); + } +} diff --git a/tests/Filee.Engines.Tests/UnicodeDataTests.cs b/tests/Filee.Engines.Tests/UnicodeDataTests.cs new file mode 100644 index 00000000..1ffe462a --- /dev/null +++ b/tests/Filee.Engines.Tests/UnicodeDataTests.cs @@ -0,0 +1,131 @@ +// The layout engine's Unicode property tables (Typesetting/UnicodeData.cs, generated by +// build/generate-unicode-tables.ps1): values of known characters, defaults for unassigned code points, and agreement +// with the Unicode version .NET uses. + +using System.Globalization; +using Filee.Engines.Typesetting; + +namespace Filee.Engines.Tests; + +public class UnicodeDataTests +{ + [Theory] + [InlineData(' ', nameof(LineBreakClass.SP))] + [InlineData('a', nameof(LineBreakClass.AL))] + [InlineData('0', nameof(LineBreakClass.NU))] + [InlineData('(', nameof(LineBreakClass.OP))] + [InlineData('\n', nameof(LineBreakClass.LF))] + [InlineData(0x00A0, nameof(LineBreakClass.GL))] // no-break space + [InlineData(0x200B, nameof(LineBreakClass.ZW))] + [InlineData(0x200D, nameof(LineBreakClass.ZWJ))] + [InlineData('가', nameof(LineBreakClass.H2))] // LV syllable + [InlineData('각', nameof(LineBreakClass.H3))] // LVT syllable + [InlineData(0x1100, nameof(LineBreakClass.JL))] + [InlineData(0x1161, nameof(LineBreakClass.JV))] + [InlineData(0x11A8, nameof(LineBreakClass.JT))] + [InlineData('一', nameof(LineBreakClass.ID))] + [InlineData('。', nameof(LineBreakClass.CL))] + [InlineData('ぁ', nameof(LineBreakClass.CJ))] + [InlineData('ก', nameof(LineBreakClass.SA))] // Thai: dictionary breaking + [InlineData('א', nameof(LineBreakClass.HL))] + [InlineData(0x1B13, nameof(LineBreakClass.AK))] // Balinese ka (Aksara, Unicode 15.1) + [InlineData(0x0915, nameof(LineBreakClass.AL))] // Devanagari ka + [InlineData(0x094D, nameof(LineBreakClass.CM))] // Devanagari virama + [InlineData(0x1F600, nameof(LineBreakClass.ID))] // emoji + [InlineData(0x3FFFD, nameof(LineBreakClass.ID))] // unassigned, CJK default + [InlineData(0xE0080, nameof(LineBreakClass.XX))] // unassigned + public void Line_break_classes(int codePoint, string expected) + { + Assert.Equal(Enum.Parse(expected), UnicodeData.GetLineBreak(codePoint)); + } + + [Theory] + [InlineData('a', nameof(BidiClass.L))] + [InlineData('א', nameof(BidiClass.R))] + [InlineData('ب', nameof(BidiClass.AL))] + [InlineData('1', nameof(BidiClass.EN))] + [InlineData(0x0663, nameof(BidiClass.AN))] // Arabic-Indic three + [InlineData(0x0301, nameof(BidiClass.NSM))] + [InlineData(' ', nameof(BidiClass.WS))] + [InlineData(0x2067, nameof(BidiClass.RLI))] + [InlineData(0x2069, nameof(BidiClass.PDI))] + [InlineData(0x05FF, nameof(BidiClass.R))] // unassigned in the Hebrew block: right-to-left by default + [InlineData(0x07B2, nameof(BidiClass.AL))] // unassigned in an Arabic range + [InlineData(0x20BF, nameof(BidiClass.ET))] // bitcoin sign + [InlineData(0xE0080, nameof(BidiClass.BN))] // unassigned default ignorable + public void Bidi_classes(int codePoint, string expected) + { + Assert.Equal(Enum.Parse(expected), UnicodeData.GetBidiClass(codePoint)); + } + + [Theory] + [InlineData('A', nameof(EastAsianWidth.Narrow))] + [InlineData('가', nameof(EastAsianWidth.Wide))] + [InlineData('“', nameof(EastAsianWidth.Ambiguous))] + [InlineData('…', nameof(EastAsianWidth.Ambiguous))] + [InlineData('é', nameof(EastAsianWidth.Ambiguous))] + [InlineData('Ａ', nameof(EastAsianWidth.Fullwidth))] + [InlineData('ｱ', nameof(EastAsianWidth.Halfwidth))] + [InlineData('ب', nameof(EastAsianWidth.Neutral))] + [InlineData(0x2A6DF, nameof(EastAsianWidth.Wide))] + [InlineData(0x3FFFD, nameof(EastAsianWidth.Wide))] // unassigned, CJK default + public void East_asian_widths(int codePoint, string expected) + { + Assert.Equal(Enum.Parse(expected), UnicodeData.GetEastAsianWidth(codePoint)); + } + + [Theory] + [InlineData('a', nameof(UnicodeScript.Latin), "Latn")] + [InlineData('가', nameof(UnicodeScript.Hangul), "Hang")] + [InlineData(0x1100, nameof(UnicodeScript.Hangul), "Hang")] + [InlineData('一', nameof(UnicodeScript.Han), "Hani")] + [InlineData('あ', nameof(UnicodeScript.Hiragana), "Hira")] + [InlineData('ア', nameof(UnicodeScript.Katakana), "Kana")] + [InlineData('ー', nameof(UnicodeScript.Common), "Zyyy")] + [InlineData(0x0301, nameof(UnicodeScript.Inherited), "Zinh")] + [InlineData('ب', nameof(UnicodeScript.Arabic), "Arab")] + [InlineData(0x0915, nameof(UnicodeScript.Devanagari), "Deva")] + [InlineData(0x0A95, nameof(UnicodeScript.Gujarati), "Gujr")] + [InlineData('ก', nameof(UnicodeScript.Thai), "Thai")] + [InlineData(0x1C89, nameof(UnicodeScript.Cyrillic), "Cyrl")] // new in Unicode 16.0 + [InlineData(0x10940, nameof(UnicodeScript.Unknown), "Zzzz")] // Sidetic, Unicode 17.0: not in the tables yet + [InlineData(0xE000, nameof(UnicodeScript.Unknown), "Zzzz")] // private use + public void Scripts_and_their_iso_codes(int codePoint, string expected, string code) + { + var script = Enum.Parse(expected); + Assert.Equal(script, UnicodeData.GetScript(codePoint)); + Assert.Equal(code, script.IsoCode()); + } + + [Fact] + public void Code_points_outside_the_code_space_get_the_defaults() + { + Assert.Equal(LineBreakClass.XX, UnicodeData.GetLineBreak(0x110000)); + Assert.Equal(UnicodeScript.Unknown, UnicodeData.GetScript(-1)); + Assert.Equal(EastAsianWidth.Neutral, UnicodeData.GetEastAsianWidth(int.MaxValue)); + } + + [Fact] + public void Tables_describe_the_characters_dotnet_knows() + { + // Same Unicode version as .NET: what .NET calls unassigned has no script, and every code point without a script + // is unassigned, private use or a surrogate. + Assert.Equal("16.0.0", UnicodeData.Version); + for (var codePoint = 0; codePoint <= 0x10FFFF; codePoint++) + { + var category = CharUnicodeInfo.GetUnicodeCategory(codePoint); + var unknown = UnicodeData.GetScript(codePoint) == UnicodeScript.Unknown; + var expected = category is UnicodeCategory.OtherNotAssigned or UnicodeCategory.PrivateUse or UnicodeCategory.Surrogate; + if (unknown != expected) + Assert.Fail($"U+{codePoint:X4} is {category} in .NET but has the script {UnicodeData.GetScript(codePoint)}."); + } + } + + [Fact] + public void Code_points_are_read_across_surrogate_pairs() + { + Assert.Equal(0x1F600, UnicodeData.CodePointAt("a\U0001F600", 1)); + Assert.Equal(0xD800, UnicodeData.CodePointAt("\uD800a", 0)); // a lone surrogate is itself + Assert.Equal(LineBreakClass.SG, UnicodeData.GetLineBreak(0xD800)); + } +}