add filesHEADmain

1 files changed, 20 insertions, 0 deletions

diff --git a/plugins/tokenize-ja/src/lib.rs b/plugins/tokenize-ja/src/lib.rs
new file mode 100644
index 0000000..fb58250
--- /dev/null
+++ b/plugins/tokenize-ja/src/lib.rs
@@ -0,0 +1,20 @@
+use ciborium::ser::into_writer;
+use wasm_minimal_protocol::*;
+
+initiate_protocol!();
+
+mod tokenize;
+
+#[wasm_func]
+pub fn init() -> Result<Vec<u8>, anyhow::Error> {
+    tokenize::init()?;
+    Ok(Vec::new())
+}
+
+#[wasm_func]
+pub fn tokenize(text: &[u8]) -> Result<Vec<u8>, anyhow::Error> {
+    let result = tokenize::tokenize(str::from_utf8(text)?)?;
+    let mut out = Vec::new();
+    into_writer(&result, &mut out)?;
+    Ok(out)
+}