MANIFEST.in
README.md
build_bundled.sh
pyproject.toml
setup.cfg
setup.py
sentencepiece/CMakeLists.txt
sentencepiece/LICENSE
sentencepiece/README.md
sentencepiece/VERSION.txt
sentencepiece/config.h.in
sentencepiece/sentencepiece.pc.in
sentencepiece/cmake/ios.toolchain.cmake
sentencepiece/src/CMakeLists.txt
sentencepiece/src/bpe_model.cc
sentencepiece/src/bpe_model.h
sentencepiece/src/bpe_model_test.cc
sentencepiece/src/bpe_model_trainer.cc
sentencepiece/src/bpe_model_trainer.h
sentencepiece/src/bpe_model_trainer_test.cc
sentencepiece/src/builder.cc
sentencepiece/src/builder.h
sentencepiece/src/builder_test.cc
sentencepiece/src/char_model.cc
sentencepiece/src/char_model.h
sentencepiece/src/char_model_test.cc
sentencepiece/src/char_model_trainer.cc
sentencepiece/src/char_model_trainer.h
sentencepiece/src/char_model_trainer_test.cc
sentencepiece/src/common.h
sentencepiece/src/compile_charsmap_main.cc
sentencepiece/src/error.cc
sentencepiece/src/filesystem.cc
sentencepiece/src/filesystem.h
sentencepiece/src/filesystem_test.cc
sentencepiece/src/freelist.h
sentencepiece/src/freelist_test.cc
sentencepiece/src/init.h
sentencepiece/src/init_test.cc
sentencepiece/src/model_factory.cc
sentencepiece/src/model_factory.h
sentencepiece/src/model_factory_test.cc
sentencepiece/src/model_interface.cc
sentencepiece/src/model_interface.h
sentencepiece/src/model_interface_test.cc
sentencepiece/src/normalization_rule.h
sentencepiece/src/normalizer.cc
sentencepiece/src/normalizer.h
sentencepiece/src/normalizer_test.cc
sentencepiece/src/pretokenizer_for_training.cc
sentencepiece/src/pretokenizer_for_training.h
sentencepiece/src/pretokenizer_for_training_test.cc
sentencepiece/src/sentencepiece.proto
sentencepiece/src/sentencepiece_model.proto
sentencepiece/src/sentencepiece_processor.cc
sentencepiece/src/sentencepiece_processor.h
sentencepiece/src/sentencepiece_processor_test.cc
sentencepiece/src/sentencepiece_trainer.cc
sentencepiece/src/sentencepiece_trainer.h
sentencepiece/src/sentencepiece_trainer_test.cc
sentencepiece/src/spec_parser.h
sentencepiece/src/spm_decode_main.cc
sentencepiece/src/spm_encode_main.cc
sentencepiece/src/spm_export_vocab_main.cc
sentencepiece/src/spm_normalize_main.cc
sentencepiece/src/spm_train_main.cc
sentencepiece/src/test_main.cc
sentencepiece/src/testharness.cc
sentencepiece/src/testharness.h
sentencepiece/src/trainer_factory.cc
sentencepiece/src/trainer_factory.h
sentencepiece/src/trainer_factory_test.cc
sentencepiece/src/trainer_interface.cc
sentencepiece/src/trainer_interface.h
sentencepiece/src/trainer_interface_test.cc
sentencepiece/src/unicode_script.cc
sentencepiece/src/unicode_script.h
sentencepiece/src/unicode_script_map.h
sentencepiece/src/unicode_script_test.cc
sentencepiece/src/unigram_model.cc
sentencepiece/src/unigram_model.h
sentencepiece/src/unigram_model_test.cc
sentencepiece/src/unigram_model_trainer.cc
sentencepiece/src/unigram_model_trainer.h
sentencepiece/src/unigram_model_trainer_test.cc
sentencepiece/src/util.cc
sentencepiece/src/util.h
sentencepiece/src/util_test.cc
sentencepiece/src/word_model.cc
sentencepiece/src/word_model.h
sentencepiece/src/word_model_test.cc
sentencepiece/src/word_model_trainer.cc
sentencepiece/src/word_model_trainer.h
sentencepiece/src/word_model_trainer_test.cc
sentencepiece/src/builtin_pb/sentencepiece.pb.cc
sentencepiece/src/builtin_pb/sentencepiece.pb.h
sentencepiece/src/builtin_pb/sentencepiece_model.pb.cc
sentencepiece/src/builtin_pb/sentencepiece_model.pb.h
sentencepiece/third_party/CMakeLists.txt
sentencepiece/third_party/absl/LICENSE
sentencepiece/third_party/absl/container/btree_set.h
sentencepiece/third_party/absl/container/flat_hash_map.h
sentencepiece/third_party/absl/container/flat_hash_set.h
sentencepiece/third_party/absl/flags/flag.cc
sentencepiece/third_party/absl/flags/flag.h
sentencepiece/third_party/absl/flags/parse.h
sentencepiece/third_party/absl/flags/usage.h
sentencepiece/third_party/absl/strings/ascii.h
sentencepiece/third_party/absl/strings/match.h
sentencepiece/third_party/absl/strings/numbers.h
sentencepiece/third_party/absl/strings/str_cat.h
sentencepiece/third_party/absl/strings/str_format.h
sentencepiece/third_party/absl/strings/str_join.h
sentencepiece/third_party/absl/strings/str_replace.h
sentencepiece/third_party/absl/strings/str_split.h
sentencepiece/third_party/absl/strings/string_view.h
sentencepiece/third_party/absl/strings/strip.h
sentencepiece/third_party/darts_clone/LICENSE
sentencepiece/third_party/darts_clone/darts.h
sentencepiece/third_party/esaxx/LICENSE
sentencepiece/third_party/esaxx/esa.hxx
sentencepiece/third_party/esaxx/sais.hxx
sentencepiece/third_party/protobuf-lite/LICENSE
sentencepiece/third_party/protobuf-lite/arena.cc
sentencepiece/third_party/protobuf-lite/arenastring.cc
sentencepiece/third_party/protobuf-lite/bytestream.cc
sentencepiece/third_party/protobuf-lite/coded_stream.cc
sentencepiece/third_party/protobuf-lite/common.cc
sentencepiece/third_party/protobuf-lite/extension_set.cc
sentencepiece/third_party/protobuf-lite/generated_enum_util.cc
sentencepiece/third_party/protobuf-lite/generated_message_table_driven_lite.cc
sentencepiece/third_party/protobuf-lite/generated_message_util.cc
sentencepiece/third_party/protobuf-lite/implicit_weak_message.cc
sentencepiece/third_party/protobuf-lite/int128.cc
sentencepiece/third_party/protobuf-lite/io_win32.cc
sentencepiece/third_party/protobuf-lite/message_lite.cc
sentencepiece/third_party/protobuf-lite/parse_context.cc
sentencepiece/third_party/protobuf-lite/repeated_field.cc
sentencepiece/third_party/protobuf-lite/status.cc
sentencepiece/third_party/protobuf-lite/statusor.cc
sentencepiece/third_party/protobuf-lite/stringpiece.cc
sentencepiece/third_party/protobuf-lite/stringprintf.cc
sentencepiece/third_party/protobuf-lite/structurally_valid.cc
sentencepiece/third_party/protobuf-lite/strutil.cc
sentencepiece/third_party/protobuf-lite/time.cc
sentencepiece/third_party/protobuf-lite/wire_format_lite.cc
sentencepiece/third_party/protobuf-lite/zero_copy_stream.cc
sentencepiece/third_party/protobuf-lite/zero_copy_stream_impl.cc
sentencepiece/third_party/protobuf-lite/zero_copy_stream_impl_lite.cc
sentencepiece/third_party/protobuf-lite/google/protobuf/any.h
sentencepiece/third_party/protobuf-lite/google/protobuf/arena.h
sentencepiece/third_party/protobuf-lite/google/protobuf/arena_impl.h
sentencepiece/third_party/protobuf-lite/google/protobuf/arenastring.h
sentencepiece/third_party/protobuf-lite/google/protobuf/descriptor.h
sentencepiece/third_party/protobuf-lite/google/protobuf/extension_set.h
sentencepiece/third_party/protobuf-lite/google/protobuf/extension_set_inl.h
sentencepiece/third_party/protobuf-lite/google/protobuf/generated_enum_reflection.h
sentencepiece/third_party/protobuf-lite/google/protobuf/generated_enum_util.h
sentencepiece/third_party/protobuf-lite/google/protobuf/generated_message_table_driven.h
sentencepiece/third_party/protobuf-lite/google/protobuf/generated_message_table_driven_lite.h
sentencepiece/third_party/protobuf-lite/google/protobuf/generated_message_util.h
sentencepiece/third_party/protobuf-lite/google/protobuf/has_bits.h
sentencepiece/third_party/protobuf-lite/google/protobuf/implicit_weak_message.h
sentencepiece/third_party/protobuf-lite/google/protobuf/map.h
sentencepiece/third_party/protobuf-lite/google/protobuf/map_entry_lite.h
sentencepiece/third_party/protobuf-lite/google/protobuf/map_field_lite.h
sentencepiece/third_party/protobuf-lite/google/protobuf/map_type_handler.h
sentencepiece/third_party/protobuf-lite/google/protobuf/message_lite.h
sentencepiece/third_party/protobuf-lite/google/protobuf/metadata_lite.h
sentencepiece/third_party/protobuf-lite/google/protobuf/parse_context.h
sentencepiece/third_party/protobuf-lite/google/protobuf/port.h
sentencepiece/third_party/protobuf-lite/google/protobuf/port_def.inc
sentencepiece/third_party/protobuf-lite/google/protobuf/port_undef.inc
sentencepiece/third_party/protobuf-lite/google/protobuf/repeated_field.h
sentencepiece/third_party/protobuf-lite/google/protobuf/unknown_field_set.h
sentencepiece/third_party/protobuf-lite/google/protobuf/wire_format_lite.h
sentencepiece/third_party/protobuf-lite/google/protobuf/io/coded_stream.h
sentencepiece/third_party/protobuf-lite/google/protobuf/io/io_win32.h
sentencepiece/third_party/protobuf-lite/google/protobuf/io/zero_copy_stream.h
sentencepiece/third_party/protobuf-lite/google/protobuf/io/zero_copy_stream_impl.h
sentencepiece/third_party/protobuf-lite/google/protobuf/io/zero_copy_stream_impl_lite.h
sentencepiece/third_party/protobuf-lite/google/protobuf/stubs/bytestream.h
sentencepiece/third_party/protobuf-lite/google/protobuf/stubs/callback.h
sentencepiece/third_party/protobuf-lite/google/protobuf/stubs/casts.h
sentencepiece/third_party/protobuf-lite/google/protobuf/stubs/common.h
sentencepiece/third_party/protobuf-lite/google/protobuf/stubs/hash.h
sentencepiece/third_party/protobuf-lite/google/protobuf/stubs/int128.h
sentencepiece/third_party/protobuf-lite/google/protobuf/stubs/logging.h
sentencepiece/third_party/protobuf-lite/google/protobuf/stubs/macros.h
sentencepiece/third_party/protobuf-lite/google/protobuf/stubs/map_util.h
sentencepiece/third_party/protobuf-lite/google/protobuf/stubs/mutex.h
sentencepiece/third_party/protobuf-lite/google/protobuf/stubs/once.h
sentencepiece/third_party/protobuf-lite/google/protobuf/stubs/platform_macros.h
sentencepiece/third_party/protobuf-lite/google/protobuf/stubs/port.h
sentencepiece/third_party/protobuf-lite/google/protobuf/stubs/status.h
sentencepiece/third_party/protobuf-lite/google/protobuf/stubs/statusor.h
sentencepiece/third_party/protobuf-lite/google/protobuf/stubs/stl_util.h
sentencepiece/third_party/protobuf-lite/google/protobuf/stubs/stringpiece.h
sentencepiece/third_party/protobuf-lite/google/protobuf/stubs/stringprintf.h
sentencepiece/third_party/protobuf-lite/google/protobuf/stubs/strutil.h
sentencepiece/third_party/protobuf-lite/google/protobuf/stubs/time.h
src/sentencepiece/__init__.py
src/sentencepiece/_version.py
src/sentencepiece/sentencepiece.i
src/sentencepiece/sentencepiece_model_pb2.py
src/sentencepiece/sentencepiece_pb2.py
src/sentencepiece/sentencepiece_wrap.cxx
src/sentencepiece.egg-info/PKG-INFO
src/sentencepiece.egg-info/SOURCES.txt
src/sentencepiece.egg-info/dependency_links.txt
src/sentencepiece.egg-info/requires.txt
src/sentencepiece.egg-info/top_level.txt
src/sentencepiece/package_data/nfkc.bin
src/sentencepiece/package_data/nfkc_cf.bin
src/sentencepiece/package_data/nmt_nfkc.bin
src/sentencepiece/package_data/nmt_nfkc_cf.bin
test/__init__.py
test/botchan.txt
test/sentencepiece_test.py
test/test_ja_model.model
test/test_model.model