Tokenization

Code

import re

# Whitespace split
tokens = "Hello world. NLP is fun.".split()

# Regex word tokenizer
words = re.findall(r"\b\w+\b", "Hello, world! NLP rocks.")

# NLTK tokenizers
from nltk.tokenize import word_tokenize, sent_tokenize
import nltk
nltk.download("punkt_tab", quiet=True)
sents = sent_tokenize("First sentence. Second one!")
tokens = word_tokenize("Don't go there.")

# spaCy tokenization
import spacy
nlp = spacy.load("en_core_web_sm")
doc = nlp("Apple is looking at buying a startup.")
spacy_tokens = [t.text for t in doc]
lemmas = [t.lemma_ for t in doc]

# Subword tokenization with HuggingFace
from transformers import AutoTokenizer
tok = AutoTokenizer.from_pretrained("bert-base-uncased")
ids = tok.encode("Tokenization is fun.", add_special_tokens=True)
print(tok.convert_ids_to_tokens(ids))

Explanation

Tokenization splits raw text into units for downstream processing. Simple splits and regex word boundaries are fast but ignore punctuation rules, while NLTK handles contractions and sentence boundaries. spaCy adds linguistic annotations, and HuggingFace tokenizers produce the subword IDs expected by transformer models.

More NLP Snippets