Feature/analysis

accuracy/Cargo.toml

@@ -15,7 +15,7 @@ rayon = { version = "1.4.0" }
 indicatif = { version = "0.15", features = ["with_rayon"] }
 # box-format = { git = "https://github.com/bbqsrc/box", branch = "master" }
 # tempdir = "0.3.7"
-pretty_env_logger = "0.4.0"
+pretty_env_logger = "0.5.0"
 # ctor = "*"
 # gumdrop = "0.8.0"
 # thiserror = "1.0.20"

accuracy/src/main.rs

@@ -21,6 +21,7 @@ static CFG: SpellerConfig = SpellerConfig {
     beam: None,
     case_handling: Some(CaseHandlingConfig::default()),
     node_pool_size: 128,
+    continuation_marker: None,
 };
 
 fn load_words(

divvunspell-bin/Cargo.toml

@@ -16,7 +16,7 @@ serde = { version = "1.0.116", features = ["derive"] }
 serde_json = "1.0.57"
 divvunspell = { version = "1.0.0-beta.3", features = ["internal_convert", "compression"], path = "../divvunspell" }
 box-format = { version = "0.3.2", features = ["reader"], default-features = false }
-pretty_env_logger = "0.4.0"
+pretty_env_logger = "0.5.0"
 gumdrop = "0.8.0"
 anyhow = "1.0.32"
 structopt = "0.3.17"

divvunspell-bin/src/main.rs

@@ -1,9 +1,14 @@
 use std::io::{self, Read};
+use std::process;
 use std::{
     path::{Path, PathBuf},
     sync::Arc,
 };
 
+use divvunspell::speller::HfstSpeller;
+use divvunspell::transducer::hfst::HfstTransducer;
+use divvunspell::transducer::Transducer;
+use divvunspell::vfs::Fs;
 use gumdrop::Options;
 use serde::Serialize;
 
@@ -24,11 +29,15 @@ use divvunspell::{
 trait OutputWriter {
     fn write_correction(&mut self, word: &str, is_correct: bool);
     fn write_suggestions(&mut self, word: &str, suggestions: &[Suggestion]);
+    fn write_input_analyses(&mut self, word: &str, analyses: &[Suggestion]);
+    fn write_output_analyses(&mut self, word: &str, analyses: &[Suggestion]);
     fn write_predictions(&mut self, predictions: &[String]);
     fn finish(&mut self);
 }
 
-struct StdoutWriter;
+struct StdoutWriter {
+    has_continuation_marker: Option<String>,
+}
 
 impl OutputWriter for StdoutWriter {
     fn write_correction(&mut self, word: &str, is_correct: bool) {
@@ -40,8 +49,18 @@ impl OutputWriter for StdoutWriter {
     }
 
     fn write_suggestions(&mut self, _word: &str, suggestions: &[Suggestion]) {
-        for sugg in suggestions {
-            println!("{}\t\t{}", sugg.value, sugg.weight);
+        if let Some(s) = &self.has_continuation_marker {
+            for sugg in suggestions {
+                print!("{}", sugg.value);
+                if sugg.completed == Some(true) {
+                    print!("{s}");
+                }
+                println!("\t\t{}", sugg.weight);
+            }
+        } else {
+            for sugg in suggestions {
+                println!("{}\t\t{}", sugg.value, sugg.weight);
+            }
         }
         println!();
     }
@@ -51,6 +70,22 @@ impl OutputWriter for StdoutWriter {
         println!("{}", predictions.join(" "));
     }
 
+    fn write_input_analyses(&mut self, _word: &str, suggestions: &[Suggestion]) {
+        println!("Input analyses: ");
+        for sugg in suggestions {
+            println!("{}\t\t{}", sugg.value, sugg.weight);
+        }
+        println!();
+    }
+
+    fn write_output_analyses(&mut self, _word: &str, suggestions: &[Suggestion]) {
+        println!("Output analyses: ");
+        for sugg in suggestions {
+            println!("{}\t\t{}", sugg.value, sugg.weight);
+        }
+        println!();
+    }
+
     fn finish(&mut self) {}
 }
 
@@ -62,18 +97,27 @@ struct SuggestionRequest {
 }
 
 #[derive(Serialize)]
+struct AnalysisRequest {
+    word: String,
+    suggestions: Vec<Suggestion>,
+}
+
+#[derive(Default, Serialize)]
 #[serde(rename_all = "camelCase")]
 struct JsonWriter {
+    #[serde(skip_serializing_if = "Vec::is_empty")]
     suggest: Vec<SuggestionRequest>,
-    predict: Option<Vec<String>>,
+    #[serde(skip_serializing_if = "Vec::is_empty")]
+    predict: Vec<String>,
+    #[serde(skip_serializing_if = "Vec::is_empty")]
+    input_analysis: Vec<AnalysisRequest>,
+    #[serde(skip_serializing_if = "Vec::is_empty")]
+    output_analysis: Vec<AnalysisRequest>,
 }
 
 impl JsonWriter {
     pub fn new() -> JsonWriter {
-        JsonWriter {
-            suggest: vec![],
-            predict: None,
-        }
+        Self::default()
     }
 }
 
@@ -92,7 +136,21 @@ impl OutputWriter for JsonWriter {
     }
 
     fn write_predictions(&mut self, predictions: &[String]) {
-        self.predict = Some(predictions.to_vec());
+        self.predict = predictions.to_vec();
+    }
+
+    fn write_input_analyses(&mut self, word: &str, suggestions: &[Suggestion]) {
+        self.input_analysis.push(AnalysisRequest {
+            word: word.to_string(),
+            suggestions: suggestions.to_vec(),
+        })
+    }
+
+    fn write_output_analyses(&mut self, word: &str, suggestions: &[Suggestion]) {
+        self.output_analysis.push(AnalysisRequest {
+            word: word.to_string(),
+            suggestions: suggestions.to_vec(),
+        })
     }
 
     fn finish(&mut self) {
@@ -104,6 +162,7 @@ fn run(
     speller: Arc<dyn Speller + Send>,
     words: Vec<String>,
     writer: &mut dyn OutputWriter,
+    is_analyzing: bool,
     is_suggesting: bool,
     is_always_suggesting: bool,
     suggest_cfg: &SpellerConfig,
@@ -116,6 +175,18 @@ fn run(
             let suggestions = speller.clone().suggest_with_config(&word, &suggest_cfg);
             writer.write_suggestions(&word, &suggestions);
         }
+
+        if is_analyzing {
+            let input_analyses = speller
+                .clone()
+                .analyze_input_with_config(&word, &suggest_cfg);
+            writer.write_input_analyses(&word, &input_analyses);
+
+            let output_analyses = speller
+                .clone()
+                .analyze_output_with_config(&word, &suggest_cfg);
+            writer.write_output_analyses(&word, &output_analyses);
+        }
     }
 }
 #[derive(Debug, Options)]
@@ -144,18 +215,30 @@ struct SuggestArgs {
     #[options(help = "print help message")]
     help: bool,
 
-    #[options(help = "BHFST or ZHFST archive to be used", required)]
-    archive: PathBuf,
+    #[options(short = "a", help = "BHFST or ZHFST archive to be used")]
+    archive_path: Option<PathBuf>,
+
+    #[options(long = "mutator", help = "mutator to use (if archive not provided)")]
+    mutator_path: Option<PathBuf>,
+
+    #[options(long = "lexicon", help = "lexicon to use (if archive not provided)")]
+    lexicon_path: Option<PathBuf>,
 
     #[options(short = "S", help = "always show suggestions even if word is correct")]
     always_suggest: bool,
 
+    #[options(short = "A", help = "analyze words and suggestions")]
+    analyze: bool,
+
     #[options(help = "maximum weight limit for suggestions")]
     weight: Option<f32>,
 
     #[options(help = "maximum number of results")]
     nbest: Option<usize>,
 
+    #[options(help = "character for incomplete predictions")]
+    continuation_marker: Option<String>,
+
     #[options(
         no_short,
         long = "no-case-handling",
@@ -283,7 +366,7 @@ fn suggest(args: SuggestArgs) -> anyhow::Result<()> {
     if args.disable_case_handling {
         suggest_cfg.case_handling = None;
     }
-
+    suggest_cfg.continuation_marker = args.continuation_marker.clone();
     if let Some(v) = args.nbest {
         if v == 0 {
             suggest_cfg.n_best = None;
@@ -303,7 +386,9 @@ fn suggest(args: SuggestArgs) -> anyhow::Result<()> {
     let mut writer: Box<dyn OutputWriter> = if args.use_json {
         Box::new(JsonWriter::new())
     } else {
-        Box::new(StdoutWriter)
+        Box::new(StdoutWriter {
+            has_continuation_marker: args.continuation_marker,
+        })
     };
 
     let words = if args.inputs.is_empty() {
@@ -321,12 +406,26 @@ fn suggest(args: SuggestArgs) -> anyhow::Result<()> {
         args.inputs.into_iter().collect()
     };
 
-    let archive = load_archive(&args.archive)?;
-    let speller = archive.speller();
+    let speller = if let Some(archive_path) = args.archive_path {
+        let archive = load_archive(&archive_path)?;
+        let speller = archive.speller();
+        speller
+    } else if let (Some(lexicon_path), Some(mutator_path)) = (args.lexicon_path, args.mutator_path)
+    {
+        let acceptor = HfstTransducer::from_path(&Fs, lexicon_path)?;
+        let errmodel = HfstTransducer::from_path(&Fs, mutator_path)?;
+
+        HfstSpeller::new(errmodel, acceptor) as _
+    } else {
+        eprintln!("Either a BHFST or ZHFST archive must be provided, or a mutator and lexicon.");
+        process::exit(1);
+    };
+
     run(
         speller,
         words,
         &mut *writer,
+        args.analyze,
         true,
         args.always_suggest,
         &suggest_cfg,

divvunspell/Cargo.toml

@@ -13,28 +13,28 @@ crate-type = ["rlib", "staticlib", "cdylib"]
 
 [dependencies]
 libc = "0.2"
-memmap2 = "0.5.0"
+memmap2 = "0.9.4"
 byteorder = "1.3.4"
 serde = { version = "1.0.116", features = ["derive"] }
 serde_json = "1.0.57"
-serde-xml-rs = { version = "0.5.0", default-features = false }
+serde-xml-rs = { version = "0.6.0", default-features = false }
 zip = { version = "0.5", default-features = false }
 unic-segment = "0.9.0"
 unic-char-range = "0.9.0"
 unic-char-property = "0.9.0"
 unic-ucd-category = "0.9.0"
-parking_lot = "0.11.2"
-hashbrown = { version = "0.11", features = ["serde"] }
+parking_lot = "0.12.1"
+hashbrown = { version = "0.14.3", features = ["serde"] }
 lifeguard = "0.6.1"
-smol_str = { version = "0.1.16", features = ["serde"] }
+smol_str = { version = "0.2.1", features = ["serde"] }
 box-format = { version = "0.3.2", features = ["reader"], default-features = false }
-itertools = "0.10"
-strsim = "0.10.0"
+itertools = "0.12.1"
+strsim = "0.11.0"
 log = "0.4.11"
-cffi = "0.1.6"
+cffi = { path = "../../../github/cffi", optional = true } #{ git = "https://github.com/cffi-rs/cffi", optional = true }
 unic-ucd-common = "0.9.0"
 flatbuffers = { version = "0.6.1", optional = true }
-env_logger = { version = "0.9", optional = true }
+env_logger = { version = "0.11.2", optional = true }
 thiserror = "1.0.20"
 tch = { version = "0.6.1", optional = true }
 rust-bert = { version = "0.17.0", optional = true }
@@ -44,7 +44,7 @@ fs_extra = "1.2.0"
 eieio = "1.0.0"
 pathos = "0.3.0"
 language-tags = "0.3.2"
-globwalk = "0.8.1"
+globwalk = "0.9.1"
 
 [features]
 compression = ["zip/deflate"]
@@ -53,4 +53,4 @@ gpt2 = ["tch", "rust-bert", "rust_tokenizers"]
 
 # Internal features: unstable, not for external use!
 internal_convert = []
-internal_ffi = ["flatbuffers", "logging"]
+internal_ffi = ["flatbuffers", "logging", "cffi"]

divvunspell/src/archive/meta.rs

@@ -90,7 +90,7 @@ fn test_xml_parse() {
             <locale>se</locale>
             <title>Giellatekno/Divvun/UiT fst-based speller for Northern Sami</title>
             <description>This is an fst-based speller for Northern Sami. It is based
-            on the normative subset of the morphological analyser for Northern Sami.
+            on the normative subset of the morphological analyzer for Northern Sami.
             The source code can be found at:
             https://victorio.uit.no/langtech/trunk/langs/sme/
             License: GPL3+.</description>

divvunspell/src/archive/mod.rs

@@ -79,7 +79,7 @@ pub(crate) mod ffi {
     use cffi::{FromForeign, ToForeign};
     use std::error::Error;
 
-    #[cffi::marshal(return_marshaler = "cffi::ArcMarshaler::<dyn SpellerArchive + Send + Sync>")]
+    #[cffi::marshal(return_marshaler = cffi::ArcMarshaler::<dyn SpellerArchive + Send + Sync>)]
     pub extern "C" fn divvun_speller_archive_open(
         #[marshal(cffi::PathBufMarshaler)] path: std::path::PathBuf,
     ) -> Result<Arc<dyn SpellerArchive + Send + Sync>, Box<dyn Error>> {