04 - Test Intelligence - Smart Test Generation and Mutation Testing

# Smart test generation framework for AI code
import ast
import inspect
from typing import List, Dict, Any

class SmartTestGenerator:
    """Generates intelligent tests by analyzing source code"""

    def __init__(self, target_function):
        self.func = target_function
        self.source = inspect.getsource(target_function)
        self.tree = ast.parse(self.source)
        self.test_cases = []

    def generate_tests(self) -> List[Dict[str, Any]]:
        """Generates test cases based on code analysis"""
        self._generate_happy_path_tests()
        self._generate_boundary_tests()
        self._generate_error_path_tests()
        self._generate_null_tests()
        self._generate_type_confusion_tests()
        return self.test_cases

    def _generate_boundary_tests(self):
        """Generates tests for boundary values identified in code"""
        comparisons = self._extract_comparisons()
        for comp in comparisons:
            # For each numeric comparison, test: value-1, value, value+1
            if isinstance(comp["value"], (int, float)):
                val = comp["value"]
                self.test_cases.extend([
                    {"type": "boundary", "input": val - 1,
                     "description": f"Just below boundary {val}"},
                    {"type": "boundary", "input": val,
                     "description": f"At boundary {val}"},
                    {"type": "boundary", "input": val + 1,
                     "description": f"Just above boundary {val}"},
                ])

    def _generate_error_path_tests(self):
        """Generates tests for each exception handler in code"""
        for node in ast.walk(self.tree):
            if isinstance(node, ast.ExceptHandler):
                exception_type = getattr(node.type, 'id', 'Exception')
                self.test_cases.append({
                    "type": "error_path",
                    "trigger": exception_type,
                    "description": f"Test error path: {exception_type}"
                })

    def _generate_null_tests(self):
        """Generates tests with None/null input for each parameter"""
        sig = inspect.signature(self.func)
        for param_name in sig.parameters:
            if param_name == 'self':
                continue
            self.test_cases.append({
                "type": "null_input",
                "param": param_name,
                "input": None,
                "description": f"None input for {param_name}"
            })

    def _extract_comparisons(self):
        """Extracts comparison values from code"""
        comparisons = []
        for node in ast.walk(self.tree):
            if isinstance(node, ast.Compare):
                for comparator in node.comparators:
                    if isinstance(comparator, ast.Constant):
                        comparisons.append({
                            "value": comparator.value,
                            "line": node.lineno
                        })
        return comparisons
      

# Mutation testing framework for AI code
class MutationTester:
    """Applies mutations to code and verifies tests catch them"""

    MUTATION_OPERATORS = [
        ("arithmetic", lambda: [
            ("+", "-"), ("-", "+"), ("*", "/"), ("/", "*")
        ]),
        ("comparison", lambda: [
            ("==", "!="), ("!=", "=="), ("<", ">="),
            (">", "<="), ("<=", ">"), (">=", "<")
        ]),
        ("logical", lambda: [
            ("and", "or"), ("or", "and"), ("True", "False"),
            ("False", "True")
        ]),
        ("boundary", lambda: [
            ("< ", "<= "), ("<= ", "< "),
            ("> ", ">= "), (">= ", "> ")
        ]),
    ]

    def __init__(self, source_code, test_suite):
        self.source = source_code
        self.tests = test_suite
        self.results = []

    def run_mutations(self):
        """Executes all mutations and calculates mutation score"""
        total_mutations = 0
        killed_mutations = 0

        for category, operators_fn in self.MUTATION_OPERATORS:
            for original, mutated in operators_fn():
                if original in self.source:
                    total_mutations += 1
                    mutated_code = self.source.replace(original, mutated, 1)
                    if self._test_detects_mutation(mutated_code):
                        killed_mutations += 1
                        status = "KILLED"
                    else:
                        status = "SURVIVED"

                    self.results.append({
                        "category": category,
                        "original": original,
                        "mutated": mutated,
                        "status": status
                    })

        mutation_score = (killed_mutations / total_mutations * 100
                         if total_mutations > 0 else 0)
        return {
            "total_mutations": total_mutations,
            "killed": killed_mutations,
            "survived": total_mutations - killed_mutations,
            "mutation_score": round(mutation_score, 2),
            "details": self.results
        }

    def _test_detects_mutation(self, mutated_code):
        """Checks if at least one test fails with mutated code"""
        # Execute tests with mutated code
        # Returns True if at least one test fails (mutation caught)
        pass  # Implementation depends on test runner
      

# Property-based testing with Hypothesis
from hypothesis import given, strategies as st, assume, settings

# Example: testing an AI-generated discount calculation function
# The AI function might have bugs on edge cases

@given(
    price=st.floats(min_value=0.01, max_value=100000),
    discount=st.floats(min_value=0, max_value=100)
)
@settings(max_examples=1000)
def test_discount_properties(price, discount):
    """Invariant properties of discount calculation"""
    result = calculate_discount(price, discount)

    # Property 1: result is never negative
    assert result >= 0, f"Negative result: {result}"

    # Property 2: result never exceeds original price
    assert result <= price, f"Result {result} > price {price}"

    # Property 3: 0% discount does not change price
    if discount == 0:
        assert result == price

    # Property 4: 100% discount brings to zero
    if discount == 100:
        assert result == 0

    # Property 5: function is monotonic with respect to discount
    # (more discount = lower price)

@given(
    items=st.lists(st.integers(min_value=1, max_value=1000),
                   min_size=0, max_size=100)
)
def test_sort_properties(items):
    """Invariant properties of an AI sorting algorithm"""
    sorted_items = ai_sort(items)

    # Property 1: same length
    assert len(sorted_items) == len(items)

    # Property 2: same elements (permutation)
    assert sorted(sorted_items) == sorted(items)

    # Property 3: actual ordering
    for i in range(len(sorted_items) - 1):
        assert sorted_items[i] <= sorted_items[i + 1]
      

# Fuzzing framework for AI-generated APIs
import random
import string
import json

class APIFuzzer:
    """Fuzzer for AI-generated API endpoints"""

    def __init__(self, endpoint_spec):
        self.spec = endpoint_spec
        self.findings = []

    def fuzz_parameter(self, param_type, iterations=100):
        """Generates fuzzed inputs for a parameter type"""
        generators = {
            "string": self._fuzz_strings,
            "integer": self._fuzz_integers,
            "email": self._fuzz_emails,
            "json": self._fuzz_json,
        }
        generator = generators.get(param_type, self._fuzz_strings)
        return [generator() for _ in range(iterations)]

    def _fuzz_strings(self):
        """Generates problematic test strings"""
        cases = [
            "",                          # empty
            " " * 1000,                  # spaces
            "A" * 100000,                # very long
            "\x00\x01\x02",             # null bytes
            "", # XSS
            "'; DROP TABLE users; --",   # SQL injection
            "../../../etc/passwd",       # path traversal
            "{{7*7}}",       # template injection
            "\r\n\r\nHTTP/1.1 200",     # header injection
            json.dumps({"$gt": ""}),   # NoSQL injection
        ]
        return random.choice(cases)

    def _fuzz_integers(self):
        """Generates boundary integers"""
        cases = [0, -1, 1, 2**31-1, -2**31, 2**63-1, -2**63]
        return random.choice(cases)

    def _fuzz_emails(self):
        """Generates malformed emails"""
        cases = [
            "", "notanemail", "@nodomain", "user@",
            "a" * 500 + "@test.com",
            "user@" + "a" * 500 + ".com",
            "user+tag@domain.com",
            "user@[127.0.0.1]",
        ]
        return random.choice(cases)

    def run_fuzzing_campaign(self, send_request_fn):
        """Executes complete fuzzing campaign"""
        for param in self.spec["parameters"]:
            fuzzed_inputs = self.fuzz_parameter(param["type"])
            for fuzz_input in fuzzed_inputs:
                try:
                    response = send_request_fn(param["name"], fuzz_input)
                    if response.status_code == 500:
                        self.findings.append({
                            "param": param["name"],
                            "input": repr(fuzz_input),
                            "status": response.status_code,
                            "severity": "HIGH"
                        })
                except Exception as e:
                    self.findings.append({
                        "param": param["name"],
                        "input": repr(fuzz_input),
                        "error": str(e),
                        "severity": "CRITICAL"
                    })
        return self.findings