diff options
Diffstat (limited to 'plugins/tokenize-ja/src/tokenize.rs')
| -rw-r--r-- | plugins/tokenize-ja/src/tokenize.rs | 2 |
1 files changed, 1 insertions, 1 deletions
diff --git a/plugins/tokenize-ja/src/tokenize.rs b/plugins/tokenize-ja/src/tokenize.rs index f7e0643..b5c13db 100644 --- a/plugins/tokenize-ja/src/tokenize.rs +++ b/plugins/tokenize-ja/src/tokenize.rs @@ -10,7 +10,7 @@ static TOKENIZER: OnceLock<Tokenizer> = OnceLock::new(); pub fn init() -> Result<(), anyhow::Error> { let dictionary = load_dictionary("embedded://ipadic").context("failed to load dictionary")?; - let segmenter = Segmenter::new(Mode::Normal, dictionary, None); + let segmenter = Segmenter::new(Mode::Normal, dictionary, None).keep_whitespace(true); let tokenizer = Tokenizer::new(segmenter); TOKENIZER .set(tokenizer) |