04 - Object Detection vs Segmentation: Tasks and Techniques

Input image: a street with 3 people and 2 cars

┌─────────────────────────────────────────────────────────────────┐
│  IMAGE CLASSIFICATION: "street with vehicles and people"        │
│  Output: 1 label for the entire image                           │
│  Does NOT say WHERE or HOW MANY objects there are               │
└─────────────────────────────────────────────────────────────────┘

┌─────────────────────────────────────────────────────────────────┐
│  OBJECT DETECTION: 5 bounding boxes                             │
│  [person(0.95) x1,y1,x2,y2]                                    │
│  [person(0.88) x1,y1,x2,y2]                                    │
│  [person(0.91) x1,y1,x2,y2]                                    │
│  [car(0.97)    x1,y1,x2,y2]                                    │
│  [car(0.94)    x1,y1,x2,y2]                                    │
│  Knows WHERE and HOW MANY, but not the precise shape            │
└─────────────────────────────────────────────────────────────────┘

┌─────────────────────────────────────────────────────────────────┐
│  SEMANTIC SEGMENTATION: every pixel has a class label           │
│  pixel(100,200)="person", pixel(300,400)="car"                  │
│  Knows the EXACT SHAPE but does NOT separate instances          │
│  All "people" = same category, not separate identities          │
└─────────────────────────────────────────────────────────────────┘

┌─────────────────────────────────────────────────────────────────┐
│  INSTANCE SEGMENTATION: mask per object instance                │
│  person_1 = {pixels: (100,200),(101,200),...}                  │
│  person_2 = {pixels: (250,180),(251,180),...}                  │
│  Knows SHAPE and distinguishes SEPARATE INSTANCES               │
└─────────────────────────────────────────────────────────────────┘

┌─────────────────────────────────────────────────────────────────┐
│  PANOPTIC SEGMENTATION: semantic + instance combined            │
│  "things" (countable): instance per person and car              │
│  "stuff" (uncountable): semantic for road, sky, buildings       │
│  Knows EVERYTHING: shape, class, identity, background           │
└─────────────────────────────────────────────────────────────────┘

import torch
import torchvision
from torchvision.models.detection import fasterrcnn_resnet50_fpn
from torchvision.models.detection import FasterRCNN_ResNet50_FPN_Weights
from torchvision.models.detection.faster_rcnn import FastRCNNPredictor

def create_faster_rcnn(num_classes: int) -> torch.nn.Module:
    """
    Faster R-CNN with pre-trained ResNet-50 + FPN backbone.
    Two-stage: RPN (Region Proposal Network) + classifier.
    """
    model = fasterrcnn_resnet50_fpn(
        weights=FasterRCNN_ResNet50_FPN_Weights.DEFAULT
    )

    # Replace box predictor for custom number of classes
    # +1 because class 0 is reserved for "background"
    in_features = model.roi_heads.box_predictor.cls_score.in_features
    model.roi_heads.box_predictor = FastRCNNPredictor(in_features, num_classes + 1)

    return model

def train_faster_rcnn(model, data_loader, num_epochs: int = 10, lr: float = 0.005):
    """Faster R-CNN training loop with built-in loss computation."""
    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
    model.to(device)
    model.train()

    optimizer = torch.optim.SGD(model.parameters(), lr=lr,
                                 momentum=0.9, weight_decay=0.0005)
    scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=3, gamma=0.1)

    for epoch in range(num_epochs):
        total_loss = 0.0
        for images, targets in data_loader:
            images = [img.to(device) for img in images]
            targets = [{k: v.to(device) for k, v in t.items()} for t in targets]

            # Faster R-CNN returns a loss dict in training mode
            loss_dict = model(images, targets)
            losses = sum(loss for loss in loss_dict.values())

            optimizer.zero_grad()
            losses.backward()
            torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)
            optimizer.step()
            total_loss += losses.item()

        scheduler.step()
        print(f"Epoch {epoch+1}/{num_epochs} | Avg Loss: {total_loss/len(data_loader):.4f}")

import torch
import torch.nn as nn
import torchvision.models.segmentation as seg_models
from torchvision.models.segmentation import DeepLabV3_ResNet50_Weights

def create_deeplabv3(num_classes: int) -> nn.Module:
    """
    DeepLabv3 with pre-trained ResNet-50 backbone.
    Uses ASPP (Atrous Spatial Pyramid Pooling) for multi-scale context.
    """
    model = seg_models.deeplabv3_resnet50(
        weights=DeepLabV3_ResNet50_Weights.DEFAULT
    )

    # Replace final classifier for custom num_classes
    model.classifier[-1] = nn.Conv2d(256, num_classes, kernel_size=1)
    model.aux_classifier[-1] = nn.Conv2d(256, num_classes, kernel_size=1)

    return model

def train_semantic_segmentation(model, data_loader, num_epochs: int = 20):
    """Training loop for semantic segmentation (mIoU optimization)."""
    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
    model.to(device)

    criterion = nn.CrossEntropyLoss(ignore_index=255)
    optimizer = torch.optim.Adam(model.parameters(), lr=1e-4, weight_decay=1e-4)
    scheduler = torch.optim.lr_scheduler.PolynomialLR(optimizer, total_iters=num_epochs)

    for epoch in range(num_epochs):
        model.train()
        total_loss = 0.0

        for images, masks in data_loader:
            images = images.to(device)
            masks = masks.long().to(device)  # [B, H, W], values 0..num_classes-1

            outputs = model(images)
            main_loss = criterion(outputs['out'], masks)
            aux_loss  = criterion(outputs['aux'], masks) * 0.4
            loss = main_loss + aux_loss

            optimizer.zero_grad()
            loss.backward()
            optimizer.step()
            total_loss += loss.item()

        scheduler.step()
        miou = compute_miou(model, data_loader, device)
        print(f"Epoch {epoch+1}/{num_epochs} | Loss: {total_loss/len(data_loader):.4f} | mIoU: {miou:.3f}")

def compute_miou(model, data_loader, device, num_classes: int = 21) -> float:
    """Computes Mean IoU - the standard metric for semantic segmentation."""
    model.eval()
    intersection = torch.zeros(num_classes, device=device)
    union = torch.zeros(num_classes, device=device)

    with torch.no_grad():
        for images, masks in data_loader:
            images, masks = images.to(device), masks.long().to(device)
            preds = model(images)['out'].argmax(dim=1)

            for cls in range(num_classes):
                pred_cls = preds == cls
                true_cls = masks == cls
                intersection[cls] += (pred_cls & true_cls).sum()
                union[cls] += (pred_cls | true_cls).sum()

    iou = intersection / (union + 1e-10)
    return float(iou[union > 0].mean())

import torch
import torchvision
from torchvision.models.detection import maskrcnn_resnet50_fpn
from torchvision.models.detection import MaskRCNN_ResNet50_FPN_Weights
from torchvision.models.detection.mask_rcnn import MaskRCNNPredictor
from torchvision.models.detection.faster_rcnn import FastRCNNPredictor

def create_mask_rcnn(num_classes: int) -> torch.nn.Module:
    """
    Mask R-CNN: Faster R-CNN + Mask Head.
    Output per instance: bbox + class + 28x28 binary mask.
    """
    model = maskrcnn_resnet50_fpn(
        weights=MaskRCNN_ResNet50_FPN_Weights.DEFAULT
    )

    # Replace box predictor
    in_features_box = model.roi_heads.box_predictor.cls_score.in_features
    model.roi_heads.box_predictor = FastRCNNPredictor(in_features_box, num_classes + 1)

    # Replace mask predictor
    in_features_mask = model.roi_heads.mask_predictor.conv5_mask.in_channels
    model.roi_heads.mask_predictor = MaskRCNNPredictor(
        in_features_mask, 256, num_classes + 1
    )

    return model

def prepare_instance_target(boxes: list, labels: list, masks: list) -> dict:
    """
    Prepares the target dict required by Mask R-CNN.
    masks: list of boolean arrays [H, W] per instance.
    """
    return {
        'boxes':  torch.tensor(boxes,  dtype=torch.float32),
        'labels': torch.tensor(labels, dtype=torch.int64),
        'masks':  torch.tensor(masks,  dtype=torch.uint8)  # [N, H, W]
    }

Question: "What do I need to know about the image?"
    |
    ├─ Just "what objects are present"?
    │   └── IMAGE CLASSIFICATION
    │       Architectures: ResNet, EfficientNet, ViT
    │       Examples: industrial quality gate, content moderation
    │
    ├─ "Where are objects + how many"?
    │   └── OBJECT DETECTION
    │       │
    │       ├─ Need real-time speed (>30 FPS)?
    │       │   └── Single-Stage: YOLO26, RT-DETR
    │       │
    │       └─ Need maximum accuracy (small objects)?
    │           └── Two-Stage: Faster R-CNN, DETR
    │
    ├─ "Class of every pixel" (no instance separation)?
    │   └── SEMANTIC SEGMENTATION
    │       Architectures: DeepLabv3, FCN, SegFormer
    │       Examples: road analysis, medical imaging, remote sensing
    │
    ├─ "Separate each object + exact shape"?
    │   └── INSTANCE SEGMENTATION
    │       Architectures: Mask R-CNN, SOLOv2, YOLACT
    │       Examples: object counting, robotics, biology
    │
    └─ "Everything: separated objects + classified background"?
        └── PANOPTIC SEGMENTATION
            Architectures: Panoptic FPN, Mask2Former
            Examples: full autonomous driving, scene understanding

import torch
import numpy as np
from collections import defaultdict

# ---- mAP for Object Detection ----
def compute_ap(recall: np.ndarray, precision: np.ndarray) -> float:
    """
    Computes Average Precision (AP) using the 11-point interpolation.
    mAP@0.5 = mean of AP for each class at IoU threshold 0.5.
    mAP@[0.5:0.95] = mean over thresholds [0.5, 0.55, ..., 0.95]
    """
    mrec = np.concatenate(([0.0], recall, [1.0]))
    mpre = np.concatenate(([0.0], precision, [0.0]))

    # Make precision monotonically decreasing
    for i in range(mpre.size - 1, 0, -1):
        mpre[i - 1] = np.maximum(mpre[i - 1], mpre[i])

    # Find recall change points
    i = np.where(mrec[1:] != mrec[:-1])[0]

    # Area under P-R curve
    ap = np.sum((mrec[i + 1] - mrec[i]) * mpre[i + 1])
    return float(ap)

def bbox_iou(box1: np.ndarray, box2: np.ndarray) -> float:
    """
    IoU between two boxes in [x1, y1, x2, y2] format.
    Used to match predictions to ground truth boxes.
    """
    x1 = max(box1[0], box2[0])
    y1 = max(box1[1], box2[1])
    x2 = min(box1[2], box2[2])
    y2 = min(box1[3], box2[3])

    intersection = max(0, x2 - x1) * max(0, y2 - y1)
    area1 = (box1[2] - box1[0]) * (box1[3] - box1[1])
    area2 = (box2[2] - box2[0]) * (box2[3] - box2[1])
    union = area1 + area2 - intersection

    return intersection / (union + 1e-10)

def compute_map(predictions: list, ground_truths: list,
                iou_threshold: float = 0.5,
                num_classes: int = 80) -> float:
    """
    Compute mAP@IoU_threshold over all classes.

    predictions: list of [image_id, class_id, confidence, x1, y1, x2, y2]
    ground_truths: list of [image_id, class_id, x1, y1, x2, y2]
    """
    # Group by class
    class_preds = defaultdict(list)
    class_gts   = defaultdict(list)

    for pred in predictions:
        img_id, cls, conf, *box = pred
        class_preds[cls].append((img_id, conf, box))

    for gt in ground_truths:
        img_id, cls, *box = gt
        class_gts[cls].append((img_id, box))

    aps = []
    for cls in range(num_classes):
        if cls not in class_gts:
            continue

        # Sort predictions by confidence (descending)
        preds = sorted(class_preds[cls], key=lambda x: -x[1])
        gt_by_image = defaultdict(list)
        for img_id, box in class_gts[cls]:
            gt_by_image[img_id].append({'box': box, 'matched': False})

        tp = np.zeros(len(preds))
        fp = np.zeros(len(preds))

        for i, (img_id, conf, pred_box) in enumerate(preds):
            best_iou = 0.0
            best_idx = -1

            for j, gt in enumerate(gt_by_image[img_id]):
                iou = bbox_iou(np.array(pred_box), np.array(gt['box']))
                if iou > best_iou:
                    best_iou = iou
                    best_idx = j

            if best_iou >= iou_threshold and not gt_by_image[img_id][best_idx]['matched']:
                tp[i] = 1
                gt_by_image[img_id][best_idx]['matched'] = True
            else:
                fp[i] = 1

        cumtp = np.cumsum(tp)
        cumfp = np.cumsum(fp)
        n_gt  = len(class_gts[cls])

        recall    = cumtp / (n_gt + 1e-10)
        precision = cumtp / (cumtp + cumfp + 1e-10)
        aps.append(compute_ap(recall, precision))

    return float(np.mean(aps)) if aps else 0.0

# ---- mIoU for Semantic Segmentation ----
def compute_miou_fast(preds: torch.Tensor, targets: torch.Tensor,
                       num_classes: int = 21,
                       ignore_index: int = 255) -> float:
    """
    Fast mIoU computation using confusion matrix.
    preds:   [B, H, W] - predicted class indices
    targets: [B, H, W] - ground truth class indices
    """
    mask = targets != ignore_index
    preds_flat   = preds[mask].long()
    targets_flat = targets[mask].long()

    # Confusion matrix via bincount
    conf_matrix = torch.bincount(
        num_classes * targets_flat + preds_flat,
        minlength=num_classes ** 2
    ).reshape(num_classes, num_classes).float()

    intersection = conf_matrix.diag()
    union = conf_matrix.sum(1) + conf_matrix.sum(0) - intersection
    iou = intersection / (union + 1e-10)

    # Only average over classes that appear in ground truth
    valid = conf_matrix.sum(1) > 0
    return float(iou[valid].mean())

import torch
import torch.nn as nn
import torch.nn.functional as F
import torchvision.models as models

class MultiTaskModel(nn.Module):
    """
    Shared ResNet-50 + FPN backbone with two heads:
    - Detection head (anchor-free box prediction)
    - Semantic segmentation head (pixel-wise classification)

    FPN (Feature Pyramid Network) enables multi-scale feature extraction:
    P5 (1/32 resolution) -> large objects
    P4 (1/16 resolution) -> medium objects
    P3 (1/8 resolution)  -> small objects
    P2 (1/4 resolution)  -> segmentation (highest resolution)
    """

    def __init__(self, num_det_classes: int, num_seg_classes: int):
        super().__init__()

        backbone = models.resnet50(weights=models.ResNet50_Weights.IMAGENET1K_V2)

        self.layer1 = nn.Sequential(backbone.conv1, backbone.bn1,
                                    backbone.relu, backbone.maxpool,
                                    backbone.layer1)   # 1/4 resolution
        self.layer2 = backbone.layer2                   # 1/8
        self.layer3 = backbone.layer3                   # 1/16
        self.layer4 = backbone.layer4                   # 1/32

        # FPN lateral connections (1x1 convolutions to normalize channels)
        self.fpn = nn.ModuleDict({
            'p5': nn.Conv2d(2048, 256, 1),
            'p4': nn.Conv2d(1024, 256, 1),
            'p3': nn.Conv2d(512,  256, 1),
            'p2': nn.Conv2d(256,  256, 1),
        })

        # Detection head on P3 (best for medium/small objects)
        self.det_head = nn.Sequential(
            nn.Conv2d(256, 256, 3, padding=1), nn.ReLU(inplace=True),
            nn.Conv2d(256, num_det_classes * 5, 1)  # 4 bbox coords + 1 objectness
        )

        # Segmentation head on P2 (highest resolution) with decoder
        self.seg_head = nn.Sequential(
            nn.Conv2d(256, 128, 3, padding=1), nn.ReLU(inplace=True),
            nn.ConvTranspose2d(128, 64, 4, stride=2, padding=1),
            nn.ReLU(inplace=True),
            nn.ConvTranspose2d(64, 32, 4, stride=2, padding=1),
            nn.ReLU(inplace=True),
            nn.Conv2d(32, num_seg_classes, 1)
        )

    def forward(self, x: torch.Tensor) -> dict:
        # Bottom-up pathway (backbone)
        c2 = self.layer1(x)    # 1/4
        c3 = self.layer2(c2)   # 1/8
        c4 = self.layer3(c3)   # 1/16
        c5 = self.layer4(c4)   # 1/32

        # Top-down pathway with lateral connections (FPN)
        p5 = self.fpn['p5'](c5)
        p4 = self.fpn['p4'](c4) + F.interpolate(p5, scale_factor=2)
        p3 = self.fpn['p3'](c3) + F.interpolate(p4, scale_factor=2)
        p2 = self.fpn['p2'](c2) + F.interpolate(p3, scale_factor=2)

        # Task-specific heads
        det_out = self.det_head(p3)
        seg_out = self.seg_head(p2)
        seg_out = F.interpolate(seg_out, size=x.shape[-2:],
                                mode='bilinear', align_corners=False)

        return {'detection': det_out, 'segmentation': seg_out}

def compute_multitask_loss(outputs: dict, det_targets, seg_targets,
                            det_weight: float = 1.0,
                            seg_weight: float = 0.5) -> tuple:
    """
    Balanced multi-task loss.

    det_weight and seg_weight control the relative importance of each task.
    These values must be tuned: if the detection loss is 100x larger than
    the segmentation loss, increase seg_weight proportionally.
    """
    det_loss = nn.BCEWithLogitsLoss()(outputs['detection'], det_targets)
    seg_loss = nn.CrossEntropyLoss(ignore_index=255)(
        outputs['segmentation'], seg_targets
    )
    total = det_weight * det_loss + seg_weight * seg_loss
    return total, {'det': float(det_loss), 'seg': float(seg_loss)}

import torch
import torch.nn as nn

class UncertaintyWeightedLoss(nn.Module):
    """
    Automatic loss balancing using learned uncertainty weights.
    From: "Multi-Task Learning Using Uncertainty to Weigh Losses"
    (Kendall et al., CVPR 2018)

    L_total = sum_i [ 1/(2*sigma_i^2) * L_i + log(sigma_i) ]

    sigma_i (log_sigma_i) is a learnable parameter per task.
    It automatically balances the losses without manual tuning.
    """

    def __init__(self, n_tasks: int):
        super().__init__()
        # Initialize log_sigma to 0 (sigma = 1, no initial weighting)
        self.log_sigmas = nn.Parameter(torch.zeros(n_tasks))

    def forward(self, losses: list) -> tuple:
        assert len(losses) == len(self.log_sigmas)

        weighted_losses = []
        for loss, log_sigma in zip(losses, self.log_sigmas):
            # L_weighted = loss / (2 * sigma^2) + log(sigma)
            # Numerically stable: use log_sigma instead of sigma
            precision = torch.exp(-2 * log_sigma)  # 1 / sigma^2
            weighted_losses.append(0.5 * precision * loss + log_sigma)

        total = sum(weighted_losses)
        weights = [float(torch.exp(-2 * ls)) for ls in self.log_sigmas]

        return total, weights

# Usage in training:
# uncertainty_loss = UncertaintyWeightedLoss(n_tasks=2)
# optimizer = torch.optim.AdamW(
#     list(model.parameters()) + list(uncertainty_loss.parameters()),
#     lr=1e-4
# )
# ...
# total, weights = uncertainty_loss([det_loss, seg_loss])
# print(f"Det weight: {weights[0]:.3f}, Seg weight: {weights[1]:.3f}")