From d962d14204ac9d15dbc4cb9b2b65055bcee8c5b2 Mon Sep 17 00:00:00 2001 From: nsfisis Date: Wed, 29 Oct 2025 03:07:59 +0900 Subject: update --- plugins/tokenize-ja/src/tokenize.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'plugins/tokenize-ja') diff --git a/plugins/tokenize-ja/src/tokenize.rs b/plugins/tokenize-ja/src/tokenize.rs index f7e0643..b5c13db 100644 --- a/plugins/tokenize-ja/src/tokenize.rs +++ b/plugins/tokenize-ja/src/tokenize.rs @@ -10,7 +10,7 @@ static TOKENIZER: OnceLock = OnceLock::new(); pub fn init() -> Result<(), anyhow::Error> { let dictionary = load_dictionary("embedded://ipadic").context("failed to load dictionary")?; - let segmenter = Segmenter::new(Mode::Normal, dictionary, None); + let segmenter = Segmenter::new(Mode::Normal, dictionary, None).keep_whitespace(true); let tokenizer = Tokenizer::new(segmenter); TOKENIZER .set(tokenizer) -- cgit v1.2.3-70-g09d2