# Source: content/notes/nlp/text-preprocessing.md
# Independent CPU example; use the curriculum environment.
# See /notes/ml/#example-environment or /notes/deep-learning/#example-environment.

import unicodedata
from tokenizers import Tokenizer, models, trainers, pre_tokenizers, decoders
raw = "Cafe\u0301 and x\u200dy"
normalized = unicodedata.normalize("NFC", raw)
assert "\u200d" in normalized and normalized != raw
assert len(raw.encode("utf-8")) > len(raw)
tokenizer = Tokenizer(models.BPE(unk_token="[UNK]"))
tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False)
tokenizer.decoder = decoders.ByteLevel()
trainer = trainers.BpeTrainer(vocab_size=280, special_tokens=["[UNK]"],
    initial_alphabet=pre_tokenizers.ByteLevel.alphabet())
tokenizer.train_from_iterator(["low lower newest widest", normalized, "x = 1234\n"], trainer)
restored = Tokenizer.from_str(tokenizer.to_str())
for text in [normalized, "unknown symbol: \u03a9", "  indented\n", "1234567"]:
    encoding = restored.encode(text)
    assert "[UNK]" not in encoding.tokens
    assert restored.decode(encoding.ids) == text
    assert all(0 <= start <= stop <= len(text) for start, stop in encoding.offsets)
print("joiner preservation, byte coverage and in-memory tokenizer reload passed")