04 - Object Detection vs Segmentation

Immagine input: una strada con 3 persone e 2 auto

┌─────────────────────────────────────────────────────────────────┐
│  IMAGE CLASSIFICATION: "strada con veicoli e persone"           │
│  Output: 1 label per tutta l'immagine                           │
│  Non dice WHERE ne QUANTI oggetti ci sono                        │
└─────────────────────────────────────────────────────────────────┘

┌─────────────────────────────────────────────────────────────────┐
│  OBJECT DETECTION: 5 bounding boxes                             │
│  [persona(0.95) x1,y1,x2,y2]                                   │
│  [persona(0.88) x1,y1,x2,y2]                                   │
│  [persona(0.91) x1,y1,x2,y2]                                   │
│  [auto(0.97)    x1,y1,x2,y2]                                   │
│  [auto(0.94)    x1,y1,x2,y2]                                   │
│  Sa WHERE e QUANTI, ma non la forma precisa                      │
└─────────────────────────────────────────────────────────────────┘

┌─────────────────────────────────────────────────────────────────┐
│  SEGMENTAZIONE SEMANTICA: ogni pixel ha una classe              │
│  pixel(100,200)="persona", pixel(300,400)="auto"                │
│  Sa la FORMA precisa, ma non distingue le istanze               │
│  Tutte le "persone" = stessa categoria, non identità separate   │
└─────────────────────────────────────────────────────────────────┘

┌─────────────────────────────────────────────────────────────────┐
│  SEGMENTAZIONE DI ISTANZA: maschera per ogni oggetto             │
│  persona_1 = {pixel: (100,200),(101,200),...}                  │
│  persona_2 = {pixel: (250,180),(251,180),...}                  │
│  Sa la FORMA e distingue le ISTANZE separate                     │
└─────────────────────────────────────────────────────────────────┘

┌─────────────────────────────────────────────────────────────────┐
│  SEGMENTAZIONE PANOPTICA: unione di semantica + istanza         │
│  "cose" (countable): istanza per persona e auto                  │
│  "stuff" (uncountable): semantica per strada, cielo, edifici    │
│  Sa TUTTO: forma, classe, istanza, sfondo                        │
└─────────────────────────────────────────────────────────────────┘

import torch
import torchvision
from torchvision.models.detection import fasterrcnn_resnet50_fpn
from torchvision.models.detection import FasterRCNN_ResNet50_FPN_Weights
from torchvision.models.detection.faster_rcnn import FastRCNNPredictor

def create_faster_rcnn(num_classes: int) -> torch.nn.Module:
    """
    Faster R-CNN con backbone ResNet-50 + FPN pre-addestrato.
    Two-stage: RPN (Region Proposal Network) + classificatore.
    """
    # Carica con pesi COCO pre-addestrati
    model = fasterrcnn_resnet50_fpn(
        weights=FasterRCNN_ResNet50_FPN_Weights.DEFAULT
    )

    # Sostituisce il classificatore per il numero di classi custom
    # +1 perchè la classe 0 e riservata al "background"
    in_features = model.roi_heads.box_predictor.cls_score.in_features
    model.roi_heads.box_predictor = FastRCNNPredictor(in_features, num_classes + 1)

    return model

def train_detection_model(model, data_loader, num_epochs: int = 10, lr: float = 0.005):
    """
    Training loop per Faster R-CNN.
    Il modello calcola automaticamente le loss interne (classification + bbox regression + RPN).
    """
    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
    model.to(device)
    model.train()

    optimizer = torch.optim.SGD(
        model.parameters(),
        lr=lr,
        momentum=0.9,
        weight_decay=0.0005
    )
    scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=3, gamma=0.1)

    for epoch in range(num_epochs):
        total_loss = 0.0
        for images, targets in data_loader:
            images = [img.to(device) for img in images]
            targets = [{k: v.to(device) for k, v in t.items()} for t in targets]

            # Faster R-CNN restituisce un dizionario di loss in training mode
            loss_dict = model(images, targets)
            losses = sum(loss for loss in loss_dict.values())

            optimizer.zero_grad()
            losses.backward()
            torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)
            optimizer.step()

            total_loss += losses.item()

        scheduler.step()
        avg_loss = total_loss / len(data_loader)
        print(f"Epoch {epoch+1}/{num_epochs} | Loss: {avg_loss:.4f}")

def inference_faster_rcnn(model, image_tensor: torch.Tensor,
                           score_threshold: float = 0.5) -> list[dict]:
    """Inference con Faster R-CNN - restituisce predizioni filtrate."""
    device = next(model.parameters()).device
    model.eval()

    with torch.no_grad():
        predictions = model([image_tensor.to(device)])

    results = []
    pred = predictions[0]
    for i, score in enumerate(pred['scores']):
        if score >= score_threshold:
            results.append({
                'bbox': pred['boxes'][i].tolist(),
                'score': float(score),
                'label': int(pred['labels'][i])
            })
    return results

import torch
import torch.nn as nn
import torchvision.models.segmentation as seg_models
from torchvision.models.segmentation import DeepLabV3_ResNet50_Weights

def create_deeplabv3(num_classes: int) -> nn.Module:
    """
    DeepLabv3 con backbone ResNet-50 pre-addestrato su COCO.
    Usa Atrous Spatial Pyramid Pooling (ASPP) per multi-scale context.
    """
    model = seg_models.deeplabv3_resnet50(
        weights=DeepLabV3_ResNet50_Weights.DEFAULT
    )

    # Sostituisce il classificatore finale per il numero di classi custom
    model.classifier[-1] = nn.Conv2d(
        in_channels=256,
        out_channels=num_classes,
        kernel_size=1
    )
    # Anche l'auxiliary classifier (per training stability)
    model.aux_classifier[-1] = nn.Conv2d(
        in_channels=256,
        out_channels=num_classes,
        kernel_size=1
    )

    return model

def train_semantic_segmentation(model, data_loader, num_epochs: int = 20):
    """
    Training loop per segmentazione semantica.
    Loss: CrossEntropyLoss (ignora label -1 per pixel non annotati)
    """
    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
    model.to(device)

    criterion = nn.CrossEntropyLoss(ignore_index=255)  # 255 = unlabeled pixel
    optimizer = torch.optim.Adam(model.parameters(), lr=1e-4, weight_decay=1e-4)
    scheduler = torch.optim.lr_scheduler.PolynomialLR(optimizer, total_iters=num_epochs)

    for epoch in range(num_epochs):
        model.train()
        total_loss = 0.0

        for images, masks in data_loader:
            images = images.to(device)
            masks = masks.long().to(device)  # [B, H, W] con valori 0..num_classes-1

            # DeepLabv3 restituisce dict con 'out' e 'aux'
            outputs = model(images)
            main_loss = criterion(outputs['out'], masks)
            aux_loss  = criterion(outputs['aux'], masks) * 0.4  # peso ridotto
            loss = main_loss + aux_loss

            optimizer.zero_grad()
            loss.backward()
            optimizer.step()

            total_loss += loss.item()

        scheduler.step()
        avg_loss = total_loss / len(data_loader)
        miou = compute_miou(model, data_loader, device)
        print(f"Epoch {epoch+1}/{num_epochs} | Loss: {avg_loss:.4f} | mIoU: {miou:.3f}")

def compute_miou(model, data_loader, device, num_classes: int = 21) -> float:
    """Calcola Mean IoU (metrica standard per segmentazione semantica)."""
    model.eval()
    intersection = torch.zeros(num_classes, device=device)
    union = torch.zeros(num_classes, device=device)

    with torch.no_grad():
        for images, masks in data_loader:
            images = images.to(device)
            masks = masks.long().to(device)
            preds = model(images)['out'].argmax(dim=1)  # [B, H, W]

            for cls in range(num_classes):
                pred_cls = preds == cls
                true_cls = masks == cls
                intersection[cls] += (pred_cls & true_cls).sum()
                union[cls] += (pred_cls | true_cls).sum()

    iou = intersection / (union + 1e-10)
    return float(iou[union > 0].mean())

import torch
import torchvision
from torchvision.models.detection import maskrcnn_resnet50_fpn
from torchvision.models.detection import MaskRCNN_ResNet50_FPN_Weights
from torchvision.models.detection.mask_rcnn import MaskRCNNPredictor
from torchvision.models.detection.faster_rcnn import FastRCNNPredictor

def create_mask_rcnn(num_classes: int) -> torch.nn.Module:
    """
    Mask R-CNN: Faster R-CNN + Mask Head.
    Output per ogni istanza: bbox + classe + maschera binaria 28x28.
    """
    model = maskrcnn_resnet50_fpn(
        weights=MaskRCNN_ResNet50_FPN_Weights.DEFAULT
    )

    # Sostituisce box predictor
    in_features_box = model.roi_heads.box_predictor.cls_score.in_features
    model.roi_heads.box_predictor = FastRCNNPredictor(
        in_features_box, num_classes + 1
    )

    # Sostituisce mask predictor
    in_features_mask = model.roi_heads.mask_predictor.conv5_mask.in_channels
    hidden_layer = 256
    model.roi_heads.mask_predictor = MaskRCNNPredictor(
        in_features_mask, hidden_layer, num_classes + 1
    )

    return model

def prepare_instance_target(boxes: list, labels: list, masks: list) -> dict:
    """
    Prepara il target nel formato richiesto da Mask R-CNN.
    masks: lista di array booleani [H, W] per ogni istanza.
    """
    return {
        'boxes': torch.tensor(boxes, dtype=torch.float32),
        'labels': torch.tensor(labels, dtype=torch.int64),
        'masks': torch.tensor(masks, dtype=torch.uint8)  # [N, H, W]
    }

def visualize_instance_predictions(image, predictions, score_threshold: float = 0.5):
    """
    Visualizza bounding boxes e maschere di istanza su un'immagine.
    """
    import numpy as np
    import cv2

    img = np.array(image)
    colors = [(np.random.randint(100, 255), np.random.randint(100, 255),
               np.random.randint(100, 255)) for _ in range(100)]

    pred = predictions[0]
    valid_idx = pred['scores'] >= score_threshold

    for i, (box, mask, score, label) in enumerate(zip(
        pred['boxes'][valid_idx],
        pred['masks'][valid_idx],
        pred['scores'][valid_idx],
        pred['labels'][valid_idx]
    )):
        color = colors[i % len(colors)]

        # Disegna bounding box
        x1, y1, x2, y2 = [int(c) for c in box]
        cv2.rectangle(img, (x1, y1), (x2, y2), color, 2)

        # Applica maschera semitrasparente
        mask_binary = (mask[0].numpy() > 0.5).astype(np.uint8)
        overlay = img.copy()
        overlay[mask_binary == 1] = color
        img = cv2.addWeighted(img, 0.6, overlay, 0.4, 0)

        # Label con confidence
        text = f"class {int(label)}: {float(score):.2f}"
        cv2.putText(img, text, (x1, y1-5),
                   cv2.FONT_HERSHEY_SIMPLEX, 0.5, color, 2)

    return img

Problema: "Cosa voglio sapere dell'immagine?"
    |
    ├─ Solo "che oggetti ci sono"?
    │   └── IMAGE CLASSIFICATION
    │       Architetture: ResNet, EfficientNet, ViT
    │       Esempi: quality gate industriale, filtro contenuti
    │
    ├─ "Dove sono gli oggetti + quanti sono"?
    │   └── OBJECT DETECTION
    │       │
    │       ├─ Serve velocità real-time (>30 FPS)?
    │       │   └── Single-Stage: YOLO26, RT-DETR
    │       │
    │       └─ Serve massima accuratezza (oggetti piccoli)?
    │           └── Two-Stage: Faster R-CNN, DETR
    │
    ├─ "Che classe e ogni pixel" (no distinzione istanze)?
    │   └── SEGMENTAZIONE SEMANTICA
    │       Architetture: DeepLabv3, FCN, SegFormer
    │       Esempi: analisi stradale, medica, telerilevamento
    │
    ├─ "Separare ogni oggetto + sua forma esatta"?
    │   └── SEGMENTAZIONE DI ISTANZA
    │       Architetture: Mask R-CNN, SOLOv2, YOLACT
    │       Esempi: conteggio oggetti, robotica, biologia
    │
    └─ "Tutto: oggetti separati + sfondo classificato"?
        └── SEGMENTAZIONE PANOPTICA
            Architetture: Panoptic FPN, Mask2Former
            Esempi: guida autonoma completa, scene understanding

import torch
import torch.nn as nn
import torchvision.models as models

class MultiTaskDetectionSegmentation(nn.Module):
    """
    Architettura multi-task che condivide un backbone ResNet-50 + FPN
    tra due head: detection e segmentazione semantica.
    """

    def __init__(self, num_det_classes: int, num_seg_classes: int):
        super().__init__()

        # Backbone condiviso: ResNet-50 con FPN
        backbone = models.resnet50(weights=models.ResNet50_Weights.IMAGENET1K_V2)

        # Estrae feature a più scale
        self.layer1 = nn.Sequential(backbone.conv1, backbone.bn1,
                                    backbone.relu, backbone.maxpool,
                                    backbone.layer1)   # 1/4 risoluzione
        self.layer2 = backbone.layer2                   # 1/8
        self.layer3 = backbone.layer3                   # 1/16
        self.layer4 = backbone.layer4                   # 1/32

        # FPN (Feature Pyramid Network) per multi-scale features
        self.fpn = nn.ModuleDict({
            'p5': nn.Conv2d(2048, 256, 1),
            'p4': nn.Conv2d(1024, 256, 1),
            'p3': nn.Conv2d(512,  256, 1),
            'p2': nn.Conv2d(256,  256, 1),
        })

        # Detection head (semplificato)
        self.det_head = nn.Sequential(
            nn.Conv2d(256, 256, 3, padding=1),
            nn.ReLU(inplace=True),
            nn.Conv2d(256, num_det_classes * (4 + 1), 1)
            # 4 bbox coords + 1 objectness per ogni classe
        )

        # Segmentation head (decoder con upsampling)
        self.seg_head = nn.Sequential(
            nn.Conv2d(256, 256, 3, padding=1),
            nn.ReLU(inplace=True),
            nn.ConvTranspose2d(256, 128, 4, stride=2, padding=1),  # 2x upsample
            nn.ReLU(inplace=True),
            nn.ConvTranspose2d(128, 64, 4, stride=2, padding=1),   # 4x upsample
            nn.ReLU(inplace=True),
            nn.Conv2d(64, num_seg_classes, 1)
        )

    def forward(self, x: torch.Tensor) -> dict:
        # Backbone
        c2 = self.layer1(x)    # 1/4
        c3 = self.layer2(c2)   # 1/8
        c4 = self.layer3(c3)   # 1/16
        c5 = self.layer4(c4)   # 1/32

        # FPN top-down pathway
        p5 = self.fpn['p5'](c5)
        p4 = self.fpn['p4'](c4) + nn.functional.interpolate(p5, scale_factor=2)
        p3 = self.fpn['p3'](c3) + nn.functional.interpolate(p4, scale_factor=2)
        p2 = self.fpn['p2'](c2) + nn.functional.interpolate(p3, scale_factor=2)

        # Task-specific heads
        det_output = self.det_head(p3)    # detection sul livello P3
        seg_output = self.seg_head(p2)    # segmentation su P2 (più alta risoluzione)

        # Upsample seg output a dimensione input
        seg_output = nn.functional.interpolate(
            seg_output, size=x.shape[-2:], mode='bilinear', align_corners=False
        )

        return {'detection': det_output, 'segmentation': seg_output}

def compute_multitask_loss(outputs: dict, det_targets, seg_targets) -> torch.Tensor:
    """
    Loss combinata multi-task con pesi bilanciati.
    Loss totale = w_det * L_det + w_seg * L_seg
    """
    det_criterion = nn.BCEWithLogitsLoss()
    seg_criterion = nn.CrossEntropyLoss(ignore_index=255)

    det_loss = det_criterion(outputs['detection'], det_targets)
    seg_loss = seg_criterion(outputs['segmentation'], seg_targets)

    # Pesi relativi (da tuning sperimentale)
    total_loss = 1.0 * det_loss + 0.5 * seg_loss

    return total_loss, {'det': det_loss.item(), 'seg': seg_loss.item()}