09 - Case Study - Implementing Quality Framework for AI Code (End-to-End)

# Baseline metrics collection script
class BaselineCollector:
    """Collects baseline metrics for the quality framework"""

    def __init__(self, git_client, sonar_client, incident_db):
        self.git = git_client
        self.sonar = sonar_client
        self.incidents = incident_db

    def collect_baseline(self, months=3):
        """Collects all baseline metrics"""
        return {
            "code_quality": self._collect_code_quality(),
            "defect_metrics": self._collect_defect_metrics(months),
            "incident_metrics": self._collect_incident_metrics(months),
            "review_metrics": self._collect_review_metrics(months),
            "coverage_metrics": self._collect_coverage_metrics(),
        }

    def _collect_defect_metrics(self, months):
        """Defect metrics separated by AI vs human"""
        commits = self.git.get_commits(months=months)
        ai_commits = [c for c in commits if self._is_ai_generated(c)]
        human_commits = [c for c in commits if not self._is_ai_generated(c)]

        return {
            "total_commits": len(commits),
            "ai_commits": len(ai_commits),
            "ai_ratio": round(len(ai_commits) / len(commits), 2),
            "ai_defect_rate": self._calculate_defect_rate(ai_commits),
            "human_defect_rate": self._calculate_defect_rate(human_commits),
            "ai_bugs_total": self._count_bugs(ai_commits),
            "human_bugs_total": self._count_bugs(human_commits),
        }

    def _collect_incident_metrics(self, months):
        """Production incident metrics"""
        incidents = self.incidents.get_recent(months=months)
        return {
            "total_incidents": len(incidents),
            "ai_related": len([i for i in incidents if i.ai_code_related]),
            "avg_mttr_hours": self._avg_mttr(incidents),
            "severity_distribution": self._severity_dist(incidents),
            "monthly_trend": self._monthly_trend(incidents),
        }
      

# CI/CD Pipeline implemented for FinPay
# .github/workflows/quality-framework.yml
name: AI Code Quality Framework

on:
  pull_request:
    branches: [main, develop]

jobs:
  secret-scan:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: TruffleHog Secret Scan
        uses: trufflesecurity/trufflehog@main
        with:
          extra_args: --only-verified

  quality-analysis:
    needs: secret-scan
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.11"

      - name: Run Tests with Coverage
        run: |
          pip install -r requirements.txt
          pytest tests/ --cov=src --cov-report=xml \
                 --cov-fail-under=75 -v

      - name: Semgrep SAST
        uses: returntocorp/semgrep-action@v1
        with:
          config: >-
            p/python
            p/security-audit
            .semgrep/ai-rules.yml

      - name: SonarQube Analysis
        uses: sonarsource/sonarqube-scan-action@master
        env:
          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}

      - name: Quality Gate
        uses: sonarsource/sonarqube-quality-gate-action@master
        env:
          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}

  mutation-testing:
    needs: quality-analysis
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Run Mutation Tests
        run: |
          pip install mutmut
          mutmut run --paths-to-mutate=src/ \
                     --tests-dir=tests/ \
                     --runner="pytest -x"
          mutmut results
      

# Quality framework monitoring dashboard
class QualityDashboard:
    """Real-time dashboard for quality framework monitoring"""

    def __init__(self, data_sources):
        self.sources = data_sources

    def generate_weekly_report(self):
        """Weekly report for the team lead"""
        return {
            "period": self._current_week(),
            "quality_metrics": {
                "defect_rate_ai": self._current_defect_rate("ai"),
                "defect_rate_human": self._current_defect_rate("human"),
                "defect_rate_trend": self._defect_rate_trend(),
                "coverage_new_code": self._new_code_coverage(),
                "gate_pass_rate": self._gate_pass_rate(),
                "avg_fix_time": self._avg_quality_fix_time(),
            },
            "delivery_metrics": {
                "deployment_frequency": self._deploy_frequency(),
                "lead_time": self._avg_lead_time(),
                "change_failure_rate": self._change_failure_rate(),
                "mttr": self._mttr(),
            },
            "team_metrics": {
                "pr_throughput": self._pr_throughput(),
                "review_turnaround": self._review_turnaround(),
                "first_time_quality": self._first_time_quality(),
                "developer_satisfaction": self._satisfaction_score(),
            },
            "top_issues": self._top_quality_issues(),
            "recommendations": self._generate_recommendations()
        }

    def _generate_recommendations(self):
        """Generates data-based recommendations"""
        recs = []
        if self._gate_pass_rate() < 70:
            recs.append({
                "type": "training",
                "message": "Low gate pass rate: consider "
                           "AI code quality training for the team"
            })
        if self._new_code_coverage() < 75:
            recs.append({
                "type": "process",
                "message": "Coverage below target: implement "
                           "test-first for complex AI prompts"
            })
        return recs
      

# Quality framework ROI calculation
def calculate_framework_roi():
    """ROI calculation for quality framework implementation"""
    hourly_rate = 75  # EUR/hour, average company cost

    # Implementation cost
    implementation_hours = 640  # 2 people x 8 weeks x 40h
    implementation_cost = implementation_hours * hourly_rate  # 48,000 EUR

    # Monthly savings
    monthly_savings = {
        # Incident reduction: from 8.3 to 3.1, saving 5.2 incidents/month
        # Average incident cost: 4.2h MTTR x 3 people involved
        "incident_reduction": 5.2 * 4.2 * 3 * hourly_rate,  # 4,914 EUR/month

        # Review time saved: from 6.8h to 3.4h per PR
        # ~30 PRs/month with AI code
        "review_time_saved": 30 * 3.4 * hourly_rate,  # 7,650 EUR/month

        # Rework reduction: -40% of bug fixing time
        # Estimate: 80 hours/month bug fixing before, now 48 hours
        "rework_reduction": 32 * hourly_rate,  # 2,400 EUR/month

        # Revenue loss avoided: -63% incidents x average impact
        "revenue_protected": 5.2 * 2500,  # 13,000 EUR/month (estimate)
    }

    total_monthly_savings = sum(monthly_savings.values())

    return {
        "implementation_cost": implementation_cost,
        "monthly_savings": total_monthly_savings,
        "payback_weeks": round(implementation_cost /
                               (total_monthly_savings / 4.33), 1),
        "annual_roi": round(
            (total_monthly_savings * 12 - implementation_cost) /
            implementation_cost * 100, 1
        ),
        "savings_breakdown": monthly_savings
    }

# Result:
# implementation_cost: 48,000 EUR
# monthly_savings: ~27,964 EUR
# payback_weeks: ~7.5 weeks
# annual_roi: ~599%