01 - NLP Fundamentals: Tokenization, Embeddings and Transformers

import re
import unicodedata

def normalize_text(text: str) -> str:
    """Basic text normalization."""
    # Lowercasing
    text = text.lower()

    # Remove accents (optional, NOT recommended for Italian)
    # text = unicodedata.normalize('NFKD', text)

    # Remove punctuation
    text = re.sub(r'[^\w\s]', '', text)

    # Remove multiple spaces
    text = re.sub(r'\s+', ' ', text).strip()

    return text

text = "NLP is AMAZING! It analyzes text in 50+ languages."
print(normalize_text(text))
# Output: "nlp is amazing it analyzes text in 50 languages"

# Approach 1: NLTK
from nltk.corpus import stopwords
import nltk
nltk.download('stopwords')

stop_en = set(stopwords.words('english'))
stop_it = set(stopwords.words('italian'))
print(f"English stopwords: {len(stop_en)}")  # ~179
print(f"Italian stopwords: {len(stop_it)}")  # ~279

text = "the cat eats the fish on the table"
tokens = text.split()
filtered = [t for t in tokens if t not in stop_en]
print(filtered)
# Output: ['cat', 'eats', 'fish', 'table']

# Approach 2: spaCy (more complete)
import spacy
nlp = spacy.load("en_core_web_sm")

doc = nlp("the cat eats the fish on the table")
filtered_spacy = [token.text for token in doc if not token.is_stop]
print(filtered_spacy)
# Output: ['cat', 'eats', 'fish', 'table']

from nltk.stem import PorterStemmer
from nltk.stem.snowball import SnowballStemmer
import spacy

# English Stemming with Porter
stemmer_en = PorterStemmer()
words_en = ["running", "runs", "ran", "easily", "fairly"]
stems_en = [stemmer_en.stem(w) for w in words_en]
print(dict(zip(words_en, stems_en)))
# {'running': 'run', 'runs': 'run', 'ran': 'ran',
#  'easily': 'easili', 'fairly': 'fairli'}

# Italian Stemming with Snowball
stemmer_it = SnowballStemmer("italian")
words_it = ["mangiando", "mangiare", "mangiato", "bellissimo"]
stems_it = [stemmer_it.stem(w) for w in words_it]
print(dict(zip(words_it, stems_it)))
# {'mangiando': 'mang', 'mangiare': 'mang',
#  'mangiato': 'mang', 'bellissimo': 'bellissim'}

# Lemmatization with spaCy
nlp = spacy.load("en_core_web_sm")
doc = nlp("The girls were eating the most beautiful apples")
for token in doc:
    print(f"  {token.text:15s} -> {token.lemma_:15s} ({token.pos_})")
# The             -> the             (DET)
# girls           -> girl            (NOUN)
# were            -> be              (AUX)
# eating          -> eat             (VERB)
# the             -> the             (DET)
# most            -> most            (ADV)
# beautiful       -> beautiful       (ADJ)
# apples          -> apple           (NOUN)

# Naive approach: split by whitespace
text = "Artificial intelligence is changing the world"
tokens_naive = text.split()
print(tokens_naive)
# ['Artificial', 'intelligence', 'is', 'changing', 'the', 'world']

# Better approach: spaCy (handles contractions, punctuation)
import spacy
nlp = spacy.load("en_core_web_sm")
doc = nlp("It's a well-known fact that NLP isn't trivial")
tokens_spacy = [token.text for token in doc]
print(tokens_spacy)
# ['It', "'s", 'a', 'well', '-', 'known', 'fact', 'that',
#  'NLP', 'is', "n't", 'trivial']
# spaCy correctly handles contractions like "isn't" -> "is" + "n't"

from transformers import AutoTokenizer

# Example text
text = "Artificial intelligence is revolutionizing the world"

# --- BERT (WordPiece) ---
bert_tok = AutoTokenizer.from_pretrained("bert-base-uncased")
bert_tokens = bert_tok.tokenize(text)
bert_ids = bert_tok.encode(text)
print("BERT tokens:", bert_tokens)
print("BERT IDs:   ", bert_ids)
# BERT tokens: ['artificial', 'intelligence', 'is',
#               'revolution', '##izing', 'the', 'world']
# Note: "revolutionizing" is split into "revolution" + "##izing"

# --- GPT-2 (BPE) ---
gpt2_tok = AutoTokenizer.from_pretrained("gpt2")
gpt2_tokens = gpt2_tok.tokenize(text)
gpt2_ids = gpt2_tok.encode(text)
print("\nGPT-2 tokens:", gpt2_tokens)
print("GPT-2 IDs:   ", gpt2_ids)
# GPT-2 tokens: ['Art', 'ificial', ' intelligence', ' is',
#                ' revolution', 'izing', ' the', ' world']

# --- Comparison ---
print(f"\nBERT:  {len(bert_tokens)} tokens")
print(f"GPT-2: {len(gpt2_tokens)} tokens")

from transformers import AutoTokenizer

# Italian BERT tokenizer
tok = AutoTokenizer.from_pretrained("dbmdz/bert-base-italian-cased")

# Vocabulary size
print(f"Italian BERT vocabulary: {tok.vocab_size} tokens")
# Output: Italian BERT vocabulary: 31102 tokens

# Special tokens
print(f"[CLS] = {tok.cls_token} (ID: {tok.cls_token_id})")
print(f"[SEP] = {tok.sep_token} (ID: {tok.sep_token_id})")
print(f"[UNK] = {tok.unk_token} (ID: {tok.unk_token_id})")
print(f"[MASK] = {tok.mask_token} (ID: {tok.mask_token_id})")

# Compare English vs Italian tokenization
text_it = "L'intelligenza artificiale sta rivoluzionando il mondo"
text_en = "Artificial intelligence is revolutionizing the world"

it_tokens = tok.tokenize(text_it)
en_tokens = tok.tokenize(text_en)
print(f"\nItalian text: {len(it_tokens)} tokens -> {it_tokens}")
print(f"English text: {len(en_tokens)} tokens -> {en_tokens}")
# Italian BERT handles Italian text efficiently (fewer tokens)
# but splits English words into more subwords

from sklearn.feature_extraction.text import CountVectorizer

documents = [
    "the cat eats the fish",
    "the dog eats the meat",
    "the cat chases the dog"
]

vectorizer = CountVectorizer()
bow_matrix = vectorizer.fit_transform(documents)

print("Vocabulary:", vectorizer.get_feature_names_out())
# ['cat', 'chases', 'dog', 'eats', 'fish', 'meat', 'the']

print("\nBoW Matrix:")
print(bow_matrix.toarray())
# [[1, 0, 0, 1, 1, 0, 2],   # doc 1
#  [0, 0, 1, 1, 0, 1, 2],   # doc 2
#  [1, 1, 1, 0, 0, 0, 2]]   # doc 3

from sklearn.feature_extraction.text import TfidfVectorizer
import numpy as np

documents = [
    "the cat eats the fish",
    "the dog eats the meat",
    "the cat chases the dog"
]

tfidf = TfidfVectorizer()
tfidf_matrix = tfidf.fit_transform(documents)

print("Features:", tfidf.get_feature_names_out())
print("\nTF-IDF Matrix (rounded):")
print(np.round(tfidf_matrix.toarray(), 3))
# "fish" and "meat" have higher weights because they appear
# in only one document (more discriminative)

import gensim.downloader as api

# Load pre-trained word embeddings
model = api.load("word2vec-google-news-300")

# Analogy: king - man + woman = ?
result = model.most_similar(
    positive=["king", "woman"],
    negative=["man"],
    topn=3
)
print("king - man + woman =")
for word, score in result:
    print(f"  {word}: {score:.4f}")
# king - man + woman =
#   queen: 0.7118
#   monarch: 0.6189
#   princess: 0.5902

# Word similarity
print(f"\ncat ~ dog: {model.similarity('cat', 'dog'):.4f}")
print(f"cat ~ car: {model.similarity('cat', 'car'):.4f}")
# cat ~ dog: 0.7609
# cat ~ car: 0.2004

from transformers import AutoTokenizer, AutoModel
import torch
import torch.nn.functional as F

# Load BERT model
tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")
model = AutoModel.from_pretrained("bert-base-uncased")

def get_word_embedding(sentence: str, word: str):
    """Get the contextual embedding of a word in a sentence."""
    inputs = tokenizer(sentence, return_tensors="pt")
    tokens = tokenizer.tokenize(sentence)

    with torch.no_grad():
        outputs = model(**inputs)

    # outputs.last_hidden_state: [batch, seq_len, hidden_dim]
    embeddings = outputs.last_hidden_state[0]  # [seq_len, 768]

    # Find the index of the target token
    word_idx = tokens.index(word) + 1  # +1 for [CLS]
    return embeddings[word_idx]

# "bank" in different contexts
emb_river = get_word_embedding("I walked along the river bank", "bank")
emb_money = get_word_embedding("I deposited money at the bank", "bank")
emb_data  = get_word_embedding("The data bank stores records", "bank")

# Calculate cosine similarity
sim_12 = F.cosine_similarity(emb_river.unsqueeze(0), emb_money.unsqueeze(0))
sim_13 = F.cosine_similarity(emb_river.unsqueeze(0), emb_data.unsqueeze(0))
sim_23 = F.cosine_similarity(emb_money.unsqueeze(0), emb_data.unsqueeze(0))

print(f"bank(river) ~ bank(money): {sim_12.item():.4f}")
print(f"bank(river) ~ bank(data):  {sim_13.item():.4f}")
print(f"bank(money) ~ bank(data):  {sim_23.item():.4f}")
# Vectors will be DIFFERENT because BERT understands context!

from sentence_transformers import SentenceTransformer, util

# Multilingual model (supports 50+ languages including Italian)
model = SentenceTransformer("paraphrase-multilingual-MiniLM-L12-v2")

# Sentences in English
sentences = [
    "The cat sleeps on the couch",
    "A feline rests on the sofa",
    "The stock market went up today",
    "Financial markets are growing",
    "I bought a new laptop computer"
]

# Generate embeddings (1 vector per sentence, 384 dimensions)
embeddings = model.encode(sentences, convert_to_tensor=True)
print(f"Shape: {embeddings.shape}")  # [5, 384]

# Calculate similarity matrix
cosine_scores = util.cos_sim(embeddings, embeddings)

print("\nSimilarity Matrix:")
for i in range(len(sentences)):
    for j in range(i + 1, len(sentences)):
        print(f"  {sentences[i][:40]:40s} <-> {sentences[j][:40]:40s}")
        print(f"    Similarity: {cosine_scores[i][j]:.4f}")

# Expected results:
# "cat sleeps"     <-> "feline rests"    : ~0.85 (very similar)
# "stock market"   <-> "markets growing" : ~0.70 (correlated)
# "cat sleeps"     <-> "stock market"    : ~0.10 (unrelated)

import spacy
import re

class ItalianPreprocessor:
    """Preprocessing pipeline specific to the Italian language."""

    def __init__(self):
        self.nlp = spacy.load("it_core_news_lg")
        # Additional regional/informal stopwords
        self.custom_stops = {
            "cioe", "quindi", "comunque", "praticamente",
            "allora", "insomma", "magari", "ecco", "tipo",
            "boh", "mah", "vabbe", "ok", "okay"
        }

    def preprocess(self, text: str, remove_stops: bool = True,
                   lemmatize: bool = True) -> list[str]:
        """Complete preprocessing for Italian text."""
        # 1. Basic normalization
        text = text.lower()
        text = re.sub(r'http\S+|www\.\S+', '', text)  # remove URLs
        text = re.sub(r'[^\w\s\']', '', text)  # keep apostrophes
        text = re.sub(r'\d+', '', text)  # remove numbers
        text = re.sub(r'\s+', ' ', text).strip()

        # 2. spaCy analysis
        doc = self.nlp(text)

        # 3. Filtering and lemmatization
        tokens = []
        for token in doc:
            # Skip punctuation and spaces
            if token.is_punct or token.is_space:
                continue
            # Skip stopwords if requested
            if remove_stops and (token.is_stop or
                                  token.text in self.custom_stops):
                continue
            # Lemmatize or use original form
            word = token.lemma_ if lemmatize else token.text
            if len(word) > 1:  # skip single characters
                tokens.append(word)

        return tokens


# Usage example
prep = ItalianPreprocessor()

text = """L'intelligenza artificiale sta rivoluzionando
il modo in cui le aziende italiane gestiscono i loro
processi, cioe praticamente tutto sta cambiando."""

result = prep.preprocess(text)
print("Processed tokens:", result)
# ['intelligenza', 'artificiale', 'rivoluzionare', 'modo',
#  'azienda', 'italiano', 'gestire', 'processo', 'cambiare']

from sentence_transformers import SentenceTransformer, util
import torch

class SemanticSearch:
    """Semantic search engine using sentence embeddings."""

    def __init__(self, model_name: str =
                 "paraphrase-multilingual-MiniLM-L12-v2"):
        self.model = SentenceTransformer(model_name)
        self.documents: list[str] = []
        self.embeddings = None

    def index_documents(self, documents: list[str]) -> None:
        """Index documents by computing their embeddings."""
        self.documents = documents
        self.embeddings = self.model.encode(
            documents,
            convert_to_tensor=True,
            show_progress_bar=True
        )
        print(f"Indexed {len(documents)} documents")
        print(f"Embeddings shape: {self.embeddings.shape}")

    def search(self, query: str, top_k: int = 3) -> list[dict]:
        """Search for the most relevant documents for the query."""
        query_embedding = self.model.encode(
            query, convert_to_tensor=True
        )
        scores = util.cos_sim(query_embedding, self.embeddings)[0]
        top_results = torch.topk(scores, k=min(top_k, len(self.documents)))

        results = []
        for score, idx in zip(top_results.values, top_results.indices):
            results.append({
                "document": self.documents[idx],
                "score": round(score.item(), 4),
                "index": idx.item()
            })
        return results


# --- Usage Example ---
corpus = [
    "Python is a versatile and easy-to-learn programming language",
    "Machine learning allows computers to learn from data",
    "Angular is a framework for building modern web applications",
    "Carbonara pasta is a typical dish of Roman cuisine",
    "Relational databases use SQL to query data",
    "Deep learning uses neural networks with many hidden layers",
    "Rome is the capital of Italy with a thousand-year history",
    "REST APIs enable communication between web services",
    "Natural Language Processing analyzes and understands text",
    "Neapolitan pizza became a UNESCO heritage in 2017"
]

# Create search engine and index documents
search_engine = SemanticSearch()
search_engine.index_documents(corpus)

# Run some searches
queries = [
    "how to analyze natural language",
    "frontend development framework",
    "traditional Italian food"
]

for query in queries:
    print(f"\nQuery: '{query}'")
    print("-" * 60)
    results = search_engine.search(query, top_k=3)
    for i, r in enumerate(results, 1):
        print(f"  {i}. [{r['score']:.4f}] {r['document']}")