From 8b2dbea5f367bf925f2370a753dddb4c6af0cc0d Mon Sep 17 00:00:00 2001 From: nsfisis Date: Mon, 24 Nov 2025 12:07:52 +0900 Subject: add files --- plugins/tokenize-ja/src/lib.rs | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 plugins/tokenize-ja/src/lib.rs (limited to 'plugins/tokenize-ja/src/lib.rs') diff --git a/plugins/tokenize-ja/src/lib.rs b/plugins/tokenize-ja/src/lib.rs new file mode 100644 index 0000000..fb58250 --- /dev/null +++ b/plugins/tokenize-ja/src/lib.rs @@ -0,0 +1,20 @@ +use ciborium::ser::into_writer; +use wasm_minimal_protocol::*; + +initiate_protocol!(); + +mod tokenize; + +#[wasm_func] +pub fn init() -> Result, anyhow::Error> { + tokenize::init()?; + Ok(Vec::new()) +} + +#[wasm_func] +pub fn tokenize(text: &[u8]) -> Result, anyhow::Error> { + let result = tokenize::tokenize(str::from_utf8(text)?)?; + let mut out = Vec::new(); + into_writer(&result, &mut out)?; + Ok(out) +} -- cgit v1.2.3-70-g09d2