diff options
| author | nsfisis <nsfisis@gmail.com> | 2025-10-29 03:07:59 +0900 |
|---|---|---|
| committer | nsfisis <nsfisis@gmail.com> | 2025-10-29 03:07:59 +0900 |
| commit | d962d14204ac9d15dbc4cb9b2b65055bcee8c5b2 (patch) | |
| tree | d0a97f8f2b93f3548245bebcf4cfeebf4b46e5c4 /plugins/tokenize-ja/src/tokenize.rs | |
| parent | 5c19575fb057db54d804abc2b3f8cf11bfbaf5d0 (diff) | |
| download | phpstudy-180-slides-d962d14204ac9d15dbc4cb9b2b65055bcee8c5b2.tar.gz phpstudy-180-slides-d962d14204ac9d15dbc4cb9b2b65055bcee8c5b2.tar.zst phpstudy-180-slides-d962d14204ac9d15dbc4cb9b2b65055bcee8c5b2.zip | |
Diffstat (limited to 'plugins/tokenize-ja/src/tokenize.rs')
| -rw-r--r-- | plugins/tokenize-ja/src/tokenize.rs | 2 |
1 files changed, 1 insertions, 1 deletions
diff --git a/plugins/tokenize-ja/src/tokenize.rs b/plugins/tokenize-ja/src/tokenize.rs index f7e0643..b5c13db 100644 --- a/plugins/tokenize-ja/src/tokenize.rs +++ b/plugins/tokenize-ja/src/tokenize.rs @@ -10,7 +10,7 @@ static TOKENIZER: OnceLock<Tokenizer> = OnceLock::new(); pub fn init() -> Result<(), anyhow::Error> { let dictionary = load_dictionary("embedded://ipadic").context("failed to load dictionary")?; - let segmenter = Segmenter::new(Mode::Normal, dictionary, None); + let segmenter = Segmenter::new(Mode::Normal, dictionary, None).keep_whitespace(true); let tokenizer = Tokenizer::new(segmenter); TOKENIZER .set(tokenizer) |