use grapheme clusters instead of chars

fixes #35 may break something else

divvunspell/src/speller/mod.rs

@@ -5,13 +5,15 @@ use hashbrown::HashMap;
 use serde::{Deserialize, Serialize};
 use smol_str::SmolStr;
 use unic_ucd_category::GeneralCategory;
+use unic_segment::Graphemes;
 
 use self::worker::SpellerWorker;
 use crate::speller::suggestion::Suggestion;
 use crate::tokenizer::case_handling::CaseHandler;
 use crate::transducer::Transducer;
 use crate::types::{SymbolNumber, Weight};
 
+
 pub mod suggestion;
 mod worker;
 
@@ -172,7 +174,8 @@ where
         let alphabet = self.mutator().alphabet();
         let key_table = alphabet.key_table();
 
-        word.chars()
+        log::trace!("to_input_vec: {}", word);
+        Graphemes::new(word)
             .map(|ch| {
                 let s = ch.to_string();
                 key_table

divvunspell/src/speller/worker.rs

@@ -399,7 +399,7 @@ where
 
         let input_sym = alphabet_translator[self.input[input_state as usize] as usize];
         let next_lexicon_state = next_node.lexicon_state + 1;
-        log::trace!("lexicon consuming {}?",
+        log::trace!("lexicon consuming {}: {}", input_sym,
             self.speller.lexicon.alphabet().string_from_symbols(&[input_sym]));
 
         if !lexicon.has_transitions(next_lexicon_state, Some(input_sym)) {