Add API to transform into standard Korean syllables

See https://www.unicode.org/reports/tr29/#Transforming_Into_SKS

Author: Jules-Bertholet

src/lib.rs

@@ -73,6 +73,7 @@ pub use crate::quick_check::{
 };
 pub use crate::recompose::Recompositions;
 pub use crate::replace::Replacements;
+pub use crate::standardize_korean_syllables::StandardKoreanSyllables;
 pub use crate::stream_safe::StreamSafe;
 pub use crate::tables::UNICODE_VERSION;
 use core::{option, str::Chars};
@@ -86,6 +87,7 @@ mod perfect_hash;
 mod quick_check;
 mod recompose;
 mod replace;
+mod standardize_korean_syllables;
 mod stream_safe;
 
 #[rustfmt::skip]
@@ -146,6 +148,10 @@ pub trait UnicodeNormalization<I: Iterator<Item = char>> {
     /// inserted according to the Stream-Safe Text Process ([UAX15-D4](https://unicode.org/reports/tr15/#UAX15-D4))
     fn stream_safe(self) -> StreamSafe<I>;
 
+    /// An iterator over the string with Hangul choseong and jugseong filler characters inserted
+    /// to ensure that all Korean syllable blocks are in standard form according to [UAX29](https://www.unicode.org/reports/tr29/#Transforming_Into_SKS).
+    fn standard_korean_syllables(self) -> StandardKoreanSyllables<I>;
+
     /// An iterator over the string in the variant of Unicode Normalization Form KD
     /// defined by Korean Standard X 1026-1. This normalization differs from that defined by Unicode
     /// in that it will not produce nonstandard Korean jamo sequences if none were present in the input.
@@ -210,6 +216,11 @@ impl<'a> UnicodeNormalization<Chars<'a>> for &'a str {
         StreamSafe::new(self.chars())
     }
 
+    #[inline]
+    fn standard_korean_syllables(self) -> StandardKoreanSyllables<Chars<'a>> {
+        StandardKoreanSyllables::new(self.chars())
+    }
+
     #[cfg(feature = "ks_x_1026-1")]
     #[cfg_attr(docsrs, doc(cfg(feature = "ks_x_1026-1")))]
     #[inline]
@@ -265,6 +276,11 @@ impl UnicodeNormalization<option::IntoIter<char>> for char {
         StreamSafe::new(Some(self).into_iter())
     }
 
+    #[inline]
+    fn standard_korean_syllables(self) -> StandardKoreanSyllables<option::IntoIter<char>> {
+        StandardKoreanSyllables::new(Some(self).into_iter())
+    }
+
     #[cfg(feature = "ks_x_1026-1")]
     #[cfg_attr(docsrs, doc(cfg(feature = "ks_x_1026-1")))]
     #[inline]
@@ -322,6 +338,11 @@ impl<I: Iterator<Item = char>> UnicodeNormalization<I> for I {
         StreamSafe::new(self)
     }
 
+    #[inline]
+    fn standard_korean_syllables(self) -> StandardKoreanSyllables<I> {
+        StandardKoreanSyllables::new(self)
+    }
+
     #[cfg(feature = "ks_x_1026-1")]
     #[cfg_attr(docsrs, doc(cfg(feature = "ks_x_1026-1")))]
     #[inline]

src/standardize_korean_syllables.rs

@@ -0,0 +1,136 @@
+use core::iter::FusedIterator;
+
+use tinyvec::ArrayVec;
+
+use crate::normalize::hangul_constants::{N_COUNT, S_BASE, T_COUNT};
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+enum JamoKind {
+    L,
+    V,
+    T,
+}
+
+impl JamoKind {
+    fn of(c: char) -> (Option<Self>, Option<Self>) {
+        match c {
+            // L
+            '\u{1100}'..='\u{115F}' | '\u{A960}'..='\u{A97C}' => {
+                (Some(JamoKind::L), Some(JamoKind::L))
+            }
+            // V
+            '\u{1160}'..='\u{11A7}' | '\u{D7B0}'..='\u{D7C6}' => {
+                (Some(JamoKind::V), Some(JamoKind::V))
+            }
+            // T
+            '\u{11A8}'..='\u{11FF}' | '\u{D7CB}'..='\u{D7FB}' => {
+                (Some(JamoKind::T), Some(JamoKind::T))
+            }
+            // LV or LVT
+            '\u{AC00}'..='\u{D7A3}' => (
+                Some(JamoKind::L),
+                Some(if ((u32::from(c) - S_BASE) % N_COUNT) % T_COUNT == 0 {
+                    // LV
+                    JamoKind::V
+                } else {
+                    // LVT
+                    JamoKind::T
+                }),
+            ),
+            _ => (None, None),
+        }
+    }
+}
+
+/// Iterator over a string's characters, with '\u{115F}' and '\u{1160}' inserted
+/// where needed to ensure all Korean syllable blocks are in standard form
+/// by [UAX29 rules](https://www.unicode.org/reports/tr29/#Standard_Korean_Syllables).
+#[derive(Clone, Debug)]
+pub struct StandardKoreanSyllables<I> {
+    prev_end_jamo_kind: Option<JamoKind>,
+    buf: ArrayVec<[Option<char>; 3]>,
+    inner: I,
+}
+
+impl<I: Iterator<Item = char>> Iterator for StandardKoreanSyllables<I> {
+    type Item = char;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        if let Some(c) = self.buf.pop() {
+            c
+        } else {
+            let next_c = self.inner.next();
+            let prev_end_jamo_kind = self.prev_end_jamo_kind;
+            let (next_start_jamo_kind, next_end_jamo_kind) =
+                next_c.map_or((None, None), JamoKind::of);
+            self.prev_end_jamo_kind = next_end_jamo_kind;
+
+            insert_fillers(
+                next_c,
+                prev_end_jamo_kind,
+                next_start_jamo_kind,
+                &mut self.buf,
+            )
+        }
+    }
+
+    #[inline]
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        let (inner_lo, inner_hi) = self.inner.size_hint();
+        let add_factor: usize = self.buf.len();
+        (
+            inner_lo.saturating_add(add_factor),
+            inner_hi
+                .and_then(|h| h.checked_mul(3)) // T → Lf Vf T
+                .and_then(|h| h.checked_add(add_factor)),
+        )
+    }
+}
+
+impl<I: Iterator<Item = char> + FusedIterator> FusedIterator for StandardKoreanSyllables<I> {}
+
+#[inline]
+fn insert_fillers(
+    next_c: Option<char>,
+    prev_end_jamo_kind: Option<JamoKind>,
+    next_start_jamo_kind: Option<JamoKind>,
+    buf: &mut ArrayVec<[Option<char>; 3]>,
+) -> Option<char> {
+    match (prev_end_jamo_kind, next_start_jamo_kind) {
+        // Insert choseong filler before V not preceded by L or V
+        (None, Some(JamoKind::V)) | (Some(JamoKind::T), Some(JamoKind::V)) => {
+            buf.push(next_c);
+            Some('\u{115F}')
+        }
+        // Insert choseong and jungseong fillers before T preceded non-jamo
+        (None, Some(JamoKind::T)) => {
+            buf.push(next_c);
+            buf.push(Some('\u{1160}'));
+            Some('\u{115F}')
+        }
+        // Insert V filler between L and non-jamo
+        (Some(JamoKind::L), None) => {
+            buf.push(next_c);
+            Some('\u{1160}')
+        }
+        // For L followed by T, insert V filler, L filler, then another V filler
+        (Some(JamoKind::L), Some(JamoKind::T)) => {
+            buf.push(next_c);
+            buf.push(Some('\u{1160}'));
+            buf.push(Some('\u{115F}'));
+            Some('\u{1160}')
+        }
+        _ => next_c,
+    }
+}
+
+impl<I> StandardKoreanSyllables<I> {
+    #[inline]
+    pub(crate) fn new(iter: I) -> Self {
+        Self {
+            prev_end_jamo_kind: None,
+            buf: ArrayVec::new(),
+            inner: iter,
+        }
+    }
+}

tests/standard_korean_syllables.rs

@@ -0,0 +1,32 @@
+use unicode_normalization::UnicodeNormalization;
+
+macro_rules! standardize {
+    ($input: expr) => {
+        IntoIterator::into_iter($input)
+            .standard_korean_syllables()
+            .collect::<Vec<char>>()
+    };
+}
+
+/// <https://www.unicode.org/reports/tr29/#Korean_Syllable_Break_Examples>
+#[test]
+fn korean_syllable_break_examples() {
+    const L: char = '\u{1100}';
+    const L_F: char = '\u{115F}';
+    const V: char = '\u{1161}';
+    const V_F: char = '\u{1160}';
+    const T: char = '\u{11AE}';
+    const LV: char = '\u{AC00}';
+    const LVT: char = '\u{AC01}';
+
+    // LVT LV LV LVf LfV LfVfT
+    let orig = [LVT, L, V, LV, L, V_F, L_F, V, L_F, V_F, T];
+    assert_eq!(standardize!(orig), orig);
+
+    // LL TT VV TT VV LLVV
+    let orig = [L, L, T, T, V, V, T, T, V, V, L, LV, V];
+    assert_eq!(
+        standardize!(orig),
+        [L, L, V_F, L_F, V_F, T, T, L_F, V, V, T, T, L_F, V, V, L, LV, V]
+    );
+}