Index: textproc/py-tokenizers/Makefile
===================================================================
RCS file: /cvs/ports/textproc/py-tokenizers/Makefile,v
diff -u -p -u -r1.2 Makefile
--- textproc/py-tokenizers/Makefile	12 Aug 2026 19:32:48 -0000	1.2
+++ textproc/py-tokenizers/Makefile	14 Oct 2026 17:59:25 -0000
@@ -1,6 +1,7 @@
 COMMENT=	tokenizers with a focus on performance
 
 MODPY_DISTV=	0.23.1
+REVISION=	0
 DISTNAME=	tokenizers-${MODPY_DISTV}
 PKGNAME=	py-${DISTNAME}
 
@@ -19,7 +20,18 @@ MODULES=	lang/python
 MODPY_PI=	Yes
 MODPY_PYBUILD=	maturin
 
-BUILD_DEPENDS=	textproc/oniguruma
+LIB_DEPENDS=	textproc/oniguruma
+WANTLIB +=	${COMPILER_LIBCXX} c m onig
+RUN_DEPENDS=	devel/py-huggingface-hub>=0.16.4,<2.0
+
+# Let Maturin read the Python metadata outside the Rust workspace.
+do-build:
+	${MODCARGO_CARGO_RUN} build --release ${MODCARGO_BUILD_ARGS}
 
 .include "crates.inc"
+MODPY_PYTEST_ARGS =	bindings/python/tests
+
+# Documentation tutorials require datasets and large online corpora.
+MODPY_TEST_DIR = ${WRKSRC}/bindings/python
+MODPY_PYTEST_ARGS = tests -m 'not network' --ignore=tests/documentation
 .include <bsd.port.mk>
Index: textproc/py-tokenizers/pkg/PLIST
===================================================================
RCS file: /cvs/ports/textproc/py-tokenizers/pkg/PLIST,v
diff -u -p -u -r1.1.1.1 PLIST
--- textproc/py-tokenizers/pkg/PLIST	19 Jul 2026 19:15:38 -0000	1.1.1.1
+++ textproc/py-tokenizers/pkg/PLIST	14 Oct 2026 17:59:25 -0000
@@ -1,10 +1,82 @@
 lib/python${MODPY_VERSION}/site-packages/tokenizers/
+lib/python${MODPY_VERSION}/site-packages/tokenizers-${MODPY_DISTV}.dist-info/
+lib/python${MODPY_VERSION}/site-packages/tokenizers-${MODPY_DISTV}.dist-info/METADATA
+lib/python${MODPY_VERSION}/site-packages/tokenizers-${MODPY_DISTV}.dist-info/RECORD
+lib/python${MODPY_VERSION}/site-packages/tokenizers-${MODPY_DISTV}.dist-info/WHEEL
 lib/python${MODPY_VERSION}/site-packages/tokenizers/__init__.py
+lib/python${MODPY_VERSION}/site-packages/tokenizers/__init__.pyi
 lib/python${MODPY_VERSION}/site-packages/tokenizers/${MODPY_PYCACHE}/
 lib/python${MODPY_VERSION}/site-packages/tokenizers/${MODPY_PYCACHE}__init__.${MODPY_PYC_MAGIC_TAG}${MODPY_PYOEXTENSION}
 lib/python${MODPY_VERSION}/site-packages/tokenizers/${MODPY_PYCACHE}__init__.${MODPY_PYC_MAGIC_TAG}pyc
+lib/python${MODPY_VERSION}/site-packages/tokenizers/decoders/
+lib/python${MODPY_VERSION}/site-packages/tokenizers/decoders.pyi
+lib/python${MODPY_VERSION}/site-packages/tokenizers/decoders/__init__.py
+lib/python${MODPY_VERSION}/site-packages/tokenizers/decoders/${MODPY_PYCACHE}/
+lib/python${MODPY_VERSION}/site-packages/tokenizers/decoders/${MODPY_PYCACHE}__init__.${MODPY_PYC_MAGIC_TAG}${MODPY_PYOEXTENSION}
+lib/python${MODPY_VERSION}/site-packages/tokenizers/decoders/${MODPY_PYCACHE}__init__.${MODPY_PYC_MAGIC_TAG}pyc
+lib/python${MODPY_VERSION}/site-packages/tokenizers/implementations/
+lib/python${MODPY_VERSION}/site-packages/tokenizers/implementations/__init__.py
+lib/python${MODPY_VERSION}/site-packages/tokenizers/implementations/${MODPY_PYCACHE}/
+lib/python${MODPY_VERSION}/site-packages/tokenizers/implementations/${MODPY_PYCACHE}__init__.${MODPY_PYC_MAGIC_TAG}${MODPY_PYOEXTENSION}
+lib/python${MODPY_VERSION}/site-packages/tokenizers/implementations/${MODPY_PYCACHE}__init__.${MODPY_PYC_MAGIC_TAG}pyc
+lib/python${MODPY_VERSION}/site-packages/tokenizers/implementations/${MODPY_PYCACHE}base_tokenizer.${MODPY_PYC_MAGIC_TAG}${MODPY_PYOEXTENSION}
+lib/python${MODPY_VERSION}/site-packages/tokenizers/implementations/${MODPY_PYCACHE}base_tokenizer.${MODPY_PYC_MAGIC_TAG}pyc
+lib/python${MODPY_VERSION}/site-packages/tokenizers/implementations/${MODPY_PYCACHE}bert_wordpiece.${MODPY_PYC_MAGIC_TAG}${MODPY_PYOEXTENSION}
+lib/python${MODPY_VERSION}/site-packages/tokenizers/implementations/${MODPY_PYCACHE}bert_wordpiece.${MODPY_PYC_MAGIC_TAG}pyc
+lib/python${MODPY_VERSION}/site-packages/tokenizers/implementations/${MODPY_PYCACHE}byte_level_bpe.${MODPY_PYC_MAGIC_TAG}${MODPY_PYOEXTENSION}
+lib/python${MODPY_VERSION}/site-packages/tokenizers/implementations/${MODPY_PYCACHE}byte_level_bpe.${MODPY_PYC_MAGIC_TAG}pyc
+lib/python${MODPY_VERSION}/site-packages/tokenizers/implementations/${MODPY_PYCACHE}char_level_bpe.${MODPY_PYC_MAGIC_TAG}${MODPY_PYOEXTENSION}
+lib/python${MODPY_VERSION}/site-packages/tokenizers/implementations/${MODPY_PYCACHE}char_level_bpe.${MODPY_PYC_MAGIC_TAG}pyc
+lib/python${MODPY_VERSION}/site-packages/tokenizers/implementations/${MODPY_PYCACHE}sentencepiece_bpe.${MODPY_PYC_MAGIC_TAG}${MODPY_PYOEXTENSION}
+lib/python${MODPY_VERSION}/site-packages/tokenizers/implementations/${MODPY_PYCACHE}sentencepiece_bpe.${MODPY_PYC_MAGIC_TAG}pyc
+lib/python${MODPY_VERSION}/site-packages/tokenizers/implementations/${MODPY_PYCACHE}sentencepiece_unigram.${MODPY_PYC_MAGIC_TAG}${MODPY_PYOEXTENSION}
+lib/python${MODPY_VERSION}/site-packages/tokenizers/implementations/${MODPY_PYCACHE}sentencepiece_unigram.${MODPY_PYC_MAGIC_TAG}pyc
+lib/python${MODPY_VERSION}/site-packages/tokenizers/implementations/base_tokenizer.py
+lib/python${MODPY_VERSION}/site-packages/tokenizers/implementations/bert_wordpiece.py
+lib/python${MODPY_VERSION}/site-packages/tokenizers/implementations/byte_level_bpe.py
+lib/python${MODPY_VERSION}/site-packages/tokenizers/implementations/char_level_bpe.py
+lib/python${MODPY_VERSION}/site-packages/tokenizers/implementations/sentencepiece_bpe.py
+lib/python${MODPY_VERSION}/site-packages/tokenizers/implementations/sentencepiece_unigram.py
+lib/python${MODPY_VERSION}/site-packages/tokenizers/models/
+lib/python${MODPY_VERSION}/site-packages/tokenizers/models.pyi
+lib/python${MODPY_VERSION}/site-packages/tokenizers/models/__init__.py
+lib/python${MODPY_VERSION}/site-packages/tokenizers/models/${MODPY_PYCACHE}/
+lib/python${MODPY_VERSION}/site-packages/tokenizers/models/${MODPY_PYCACHE}__init__.${MODPY_PYC_MAGIC_TAG}${MODPY_PYOEXTENSION}
+lib/python${MODPY_VERSION}/site-packages/tokenizers/models/${MODPY_PYCACHE}__init__.${MODPY_PYC_MAGIC_TAG}pyc
+lib/python${MODPY_VERSION}/site-packages/tokenizers/normalizers/
+lib/python${MODPY_VERSION}/site-packages/tokenizers/normalizers.pyi
+lib/python${MODPY_VERSION}/site-packages/tokenizers/normalizers/__init__.py
+lib/python${MODPY_VERSION}/site-packages/tokenizers/normalizers/${MODPY_PYCACHE}/
+lib/python${MODPY_VERSION}/site-packages/tokenizers/normalizers/${MODPY_PYCACHE}__init__.${MODPY_PYC_MAGIC_TAG}${MODPY_PYOEXTENSION}
+lib/python${MODPY_VERSION}/site-packages/tokenizers/normalizers/${MODPY_PYCACHE}__init__.${MODPY_PYC_MAGIC_TAG}pyc
+lib/python${MODPY_VERSION}/site-packages/tokenizers/pre_tokenizers/
+lib/python${MODPY_VERSION}/site-packages/tokenizers/pre_tokenizers.pyi
+lib/python${MODPY_VERSION}/site-packages/tokenizers/pre_tokenizers/__init__.py
+lib/python${MODPY_VERSION}/site-packages/tokenizers/pre_tokenizers/${MODPY_PYCACHE}/
+lib/python${MODPY_VERSION}/site-packages/tokenizers/pre_tokenizers/${MODPY_PYCACHE}__init__.${MODPY_PYC_MAGIC_TAG}${MODPY_PYOEXTENSION}
+lib/python${MODPY_VERSION}/site-packages/tokenizers/pre_tokenizers/${MODPY_PYCACHE}__init__.${MODPY_PYC_MAGIC_TAG}pyc
+lib/python${MODPY_VERSION}/site-packages/tokenizers/processors/
+lib/python${MODPY_VERSION}/site-packages/tokenizers/processors.pyi
+lib/python${MODPY_VERSION}/site-packages/tokenizers/processors/__init__.py
+lib/python${MODPY_VERSION}/site-packages/tokenizers/processors/${MODPY_PYCACHE}/
+lib/python${MODPY_VERSION}/site-packages/tokenizers/processors/${MODPY_PYCACHE}__init__.${MODPY_PYC_MAGIC_TAG}${MODPY_PYOEXTENSION}
+lib/python${MODPY_VERSION}/site-packages/tokenizers/processors/${MODPY_PYCACHE}__init__.${MODPY_PYC_MAGIC_TAG}pyc
+lib/python${MODPY_VERSION}/site-packages/tokenizers/py.typed
 @so lib/python${MODPY_VERSION}/site-packages/tokenizers/tokenizers${MODPY_ABI3SO}.so
-lib/python${MODPY_VERSION}/site-packages/tokenizers_python-${MODPY_DISTV}.dist-info/
-lib/python${MODPY_VERSION}/site-packages/tokenizers_python-${MODPY_DISTV}.dist-info/METADATA
-lib/python${MODPY_VERSION}/site-packages/tokenizers_python-${MODPY_DISTV}.dist-info/RECORD
-lib/python${MODPY_VERSION}/site-packages/tokenizers_python-${MODPY_DISTV}.dist-info/WHEEL
+lib/python${MODPY_VERSION}/site-packages/tokenizers/tokenizers.pyi
+lib/python${MODPY_VERSION}/site-packages/tokenizers/tools/
+lib/python${MODPY_VERSION}/site-packages/tokenizers/tools/__init__.py
+lib/python${MODPY_VERSION}/site-packages/tokenizers/tools/${MODPY_PYCACHE}/
+lib/python${MODPY_VERSION}/site-packages/tokenizers/tools/${MODPY_PYCACHE}__init__.${MODPY_PYC_MAGIC_TAG}${MODPY_PYOEXTENSION}
+lib/python${MODPY_VERSION}/site-packages/tokenizers/tools/${MODPY_PYCACHE}__init__.${MODPY_PYC_MAGIC_TAG}pyc
+lib/python${MODPY_VERSION}/site-packages/tokenizers/tools/${MODPY_PYCACHE}visualizer.${MODPY_PYC_MAGIC_TAG}${MODPY_PYOEXTENSION}
+lib/python${MODPY_VERSION}/site-packages/tokenizers/tools/${MODPY_PYCACHE}visualizer.${MODPY_PYC_MAGIC_TAG}pyc
+lib/python${MODPY_VERSION}/site-packages/tokenizers/tools/visualizer-styles.css
+lib/python${MODPY_VERSION}/site-packages/tokenizers/tools/visualizer.py
+lib/python${MODPY_VERSION}/site-packages/tokenizers/trainers/
+lib/python${MODPY_VERSION}/site-packages/tokenizers/trainers.pyi
+lib/python${MODPY_VERSION}/site-packages/tokenizers/trainers/__init__.py
+lib/python${MODPY_VERSION}/site-packages/tokenizers/trainers/__init__.pyi
+lib/python${MODPY_VERSION}/site-packages/tokenizers/trainers/${MODPY_PYCACHE}/
+lib/python${MODPY_VERSION}/site-packages/tokenizers/trainers/${MODPY_PYCACHE}__init__.${MODPY_PYC_MAGIC_TAG}${MODPY_PYOEXTENSION}
+lib/python${MODPY_VERSION}/site-packages/tokenizers/trainers/${MODPY_PYCACHE}__init__.${MODPY_PYC_MAGIC_TAG}pyc