#!/usr/bin/env python3 """ Regression Test Template and Examples This file demonstrates the regression testing pattern for the Bug Tracking & Regression Prevention System. Copy the template section and adapt it for your bug fix. Related: - ../../docs/workflows/REGRESSION_PREVENTION.md - ../failure-registry-examples.jsonl - ../prevention-rules-examples.json Usage: 1. Copy the TEMPLATE section below 2. Update FAILURE_ID, description, and test cases 3. Place in tests/regression/test__regression_.py 4. Run with: python -m pytest tests/regression/test_*_regression_*.py -v """ import unittest from typing import Any, Dict, Optional # ============================================================================= # TEMPLATE - Copy and customize for new regression tests # ============================================================================= """ # tests/regression/test__regression_.py \'\'\' Regression test for FAILURE_ID: FAIL-XXX-NNN Bug: Fix: Registry: ../../.guardrails/failure-registry.jsonl This test MUST fail with the buggy code and pass with the fix. If this test ever fails, the bug has been reintroduced. \'\'\' import unittest class TestRegression(unittest.TestCase): \'\'\' Regression test for : Original bug: Root cause: Fix commit: \'\'\' def test__should_(self): \'\'\' Test that is handled correctly. This test would have caught the original bug where . \'\'\' # Arrange # Act & Assert with self.assertRaises() as ctx: self.assertIn(, str(ctx.exception)) def test__should_work(self): \'\'\'Normal operation should not be affected by the fix.\'\'\' # Arrange # Act result = # Assert self.assertEqual(result, ) if __name__ == '__main__': unittest.main() """ # ============================================================================= # EXAMPLE 1: Null Check After JSON Parse (FAIL-WEB-001) # ============================================================================= # This example shows a regression test for a real bug where parsing webhook # payload without null check caused production outage. class PaymentWebhookParser: """Parses payment webhook payloads - FIXED VERSION.""" @staticmethod def parse_amount(payload: str) -> Dict[str, Any]: """Parse payment amount from webhook payload with validation.""" if not payload: raise ValueError("Empty payload received") try: data = __import__('json').loads(payload) except __import__('json').JSONDecodeError as e: raise ValueError(f"Invalid JSON in payload: {e}") if not isinstance(data, dict): raise ValueError("Payload must be a JSON object") if 'amount' not in data: raise ValueError("Missing required field: amount") if not isinstance(data['amount'], (int, float)): raise ValueError("Amount must be a number") return data class TestPaymentRegressionFAIL_WEB_001(unittest.TestCase): """ Regression test for FAIL-WEB-001: Null check after JSON parse Original bug: JSON.parse result accessed without null check caused TypeError when webhook payload was malformed or empty. Impact: Payment processing failed for 500+ customers. Fix: Added comprehensive validation layer before property access. Fix commit: a1b2c3d4e5f6789012345678 Prevention rule: PREVENT-EX-001 """ def test_parse_empty_payload_should_raise_error(self): """ Empty payload should raise ValueError, not TypeError. Buggy code: const data = JSON.parse(payload); return data.amount; Would throw: TypeError: Cannot read property 'amount' of undefined """ with self.assertRaises(ValueError) as ctx: PaymentWebhookParser.parse_amount("") self.assertIn("Empty payload", str(ctx.exception)) def test_parse_invalid_json_should_raise_error(self): """ Invalid JSON should raise ValueError with context. Buggy code would crash with JSONDecodeError or return None causing later TypeError. """ with self.assertRaises(ValueError) as ctx: PaymentWebhookParser.parse_amount("not valid json {{ ") self.assertIn("Invalid JSON", str(ctx.exception)) def test_parse_non_object_json_should_raise_error(self): """ Valid JSON that is not an object should be rejected. Payload like "null" or "123" would pass JSON.parse but fail on property access. """ with self.assertRaises(ValueError) as ctx: PaymentWebhookParser.parse_amount("null") self.assertIn("must be a JSON object", str(ctx.exception)) def test_parse_missing_amount_should_raise_error(self): """ JSON object missing required 'amount' field should fail gracefully.""" with self.assertRaises(ValueError) as ctx: PaymentWebhookParser.parse_amount('{"currency": "USD"}') self.assertIn("Missing required field: amount", str(ctx.exception)) def test_parse_valid_payload_should_work(self): """Normal operation: valid payload should parse correctly.""" result = PaymentWebhookParser.parse_amount('{"amount": 99.99, "currency": "USD"}') self.assertEqual(result['amount'], 99.99) self.assertEqual(result['currency'], 'USD') # ============================================================================= # EXAMPLE 2: Cache Race Condition (FAIL-CACHE-001) # ============================================================================= # This example shows regression test for race condition in cache updates. class CacheManager: """Thread-safe cache manager - FIXED VERSION.""" def __init__(self): self._cache: Dict[str, Any] = {} self._locks: Dict[str, Any] = {} self._global_lock = __import__('threading').Lock() def _get_lock(self, key: str): """Get or create lock for key.""" with self._global_lock: if key not in self._locks: self._locks[key] = __import__('threading').Lock() return self._locks[key] def atomic_increment(self, key: str) -> int: """Atomically increment counter - thread safe.""" with self._get_lock(key): current = self._cache.get(key, 0) new_value = current + 1 self._cache[key] = new_value return new_value # BUGGY VERSION (for testing the test): # def atomic_increment(self, key: str) -> int: # """Non-atomic increment - has race condition.""" # current = self._cache.get(key, 0) # Race: two threads read same value # new_value = current + 1 # self._cache[key] = new_value # Race: last write wins # return new_value class TestCacheRegressionFAIL_CACHE_001(unittest.TestCase): """ Regression test for FAIL-CACHE-001: Cache race condition Original bug: Non-atomic read-modify-write cycle caused stale data under concurrent load. Two threads would read same value, both increment, and both write - one update lost. Impact: Inventory counts wrong, overselling products. Fix: Added per-key locking for atomic operations. Fix commit: c3d4e5f6a7b8901234567890 Prevention rule: PREVENT-EX-005 """ def test_concurrent_increments_should_not_lose_updates(self): """ Multiple threads incrementing same key should all be counted. Buggy code would lose updates: Thread A: read 0 -> increment -> write 1 Thread B: read 0 -> increment -> write 1 (A's update lost!) Fixed code: Thread A: acquire lock -> read 0 -> increment -> write 1 -> release Thread B: acquire lock -> read 1 -> increment -> write 2 -> release """ import threading import time cache = CacheManager() key = "counter" num_threads = 10 increments_per_thread = 100 def increment_many(): for _ in range(increments_per_thread): cache.atomic_increment(key) time.sleep(0.001) # Increase chance of race threads = [ threading.Thread(target=increment_many) for _ in range(num_threads) ] for t in threads: t.start() for t in threads: t.join() expected = num_threads * increments_per_thread actual = cache._cache.get(key, 0) self.assertEqual( actual, expected, f"Race condition detected: expected {expected}, got {actual}. " f"Lost {expected - actual} updates!" ) def test_separate_keys_should_not_block_each_other(self): """Different keys can be updated concurrently without blocking.""" import threading import time cache = CacheManager() results = {} def increment_key(key: str, count: int): for _ in range(count): cache.atomic_increment(key) results[key] = cache._cache.get(key, 0) threads = [ threading.Thread(target=increment_key, args=(f"key_{i}", 10)) for i in range(5) ] start_time = time.time() for t in threads: t.start() for t in threads: t.join() elapsed = time.time() - start_time # All keys should have correct values for i in range(5): self.assertEqual(results.get(f"key_{i}"), 10) # Should complete in reasonable time (parallel, not sequential) self.assertLess(elapsed, 2.0, "Operations appear sequential, not parallel") # ============================================================================= # EXAMPLE 3: Environment Variable Validation (FAIL-CFG-001) # ============================================================================= # This example shows regression test for missing env var validation. class ConfigLoader: """Configuration loader with validation - FIXED VERSION.""" REQUIRED_VARS = ['DATABASE_URL', 'API_KEY', 'SECRET_KEY'] @classmethod def load(cls) -> Dict[str, str]: """Load configuration with validation.""" import os config = {} missing = [] for var in cls.REQUIRED_VARS: value = os.environ.get(var) if not value: missing.append(var) else: config[var] = value if missing: raise RuntimeError( f"Missing required environment variables: {', '.join(missing)}. " f"Please set them before starting the application." ) return config class TestConfigRegressionFAIL_CFG_001(unittest.TestCase): """ Regression test for FAIL-CFG-001: Missing env var validation Original bug: Service crashed on startup with KeyError when DATABASE_URL environment variable was not set. Error only manifested on first database connection attempt. Impact: Production deployment rolled back, 2 hour outage. Fix: Added startup validation with clear error messages. Fix commit: d4e5f6a7b8c9012345678901 Prevention rule: PREVENT-EX-003 """ def setUp(self): """Save original environment.""" import os self.original_env = dict(os.environ) def tearDown(self): """Restore original environment.""" import os os.environ.clear() os.environ.update(self.original_env) def test_missing_database_url_should_raise_clear_error(self): """ Missing DATABASE_URL should give clear error at startup. Buggy code would crash later with: KeyError: 'DATABASE_URL' Fixed code gives: RuntimeError: Missing required environment variables: DATABASE_URL """ import os # Clear required vars for var in ConfigLoader.REQUIRED_VARS: if var in os.environ: del os.environ[var] with self.assertRaises(RuntimeError) as ctx: ConfigLoader.load() error_msg = str(ctx.exception) self.assertIn("DATABASE_URL", error_msg) self.assertIn("Missing required", error_msg) self.assertIn("environment variables", error_msg) def test_multiple_missing_vars_should_list_all(self): """Error should list ALL missing vars, not just first one.""" import os # Clear all required vars for var in ConfigLoader.REQUIRED_VARS: if var in os.environ: del os.environ[var] with self.assertRaises(RuntimeError) as ctx: ConfigLoader.load() error_msg = str(ctx.exception) # Should mention all missing variables for var in ConfigLoader.REQUIRED_VARS: self.assertIn(var, error_msg) def test_valid_environment_should_load_successfully(self): """Normal operation with all vars set.""" import os # Set all required vars os.environ['DATABASE_URL'] = 'postgresql://localhost/db' os.environ['API_KEY'] = 'test-api-key' os.environ['SECRET_KEY'] = 'test-secret-key' config = ConfigLoader.load() self.assertEqual(config['DATABASE_URL'], 'postgresql://localhost/db') self.assertEqual(config['API_KEY'], 'test-api-key') self.assertEqual(config['SECRET_KEY'], 'test-secret-key') # ============================================================================= # Running the tests # ============================================================================= if __name__ == '__main__': # Run with verbose output unittest.main(verbosity=2) # Or from command line: # python -m pytest tests/regression/ -v # python -m pytest tests/regression/test_*_regression_*.py -v --tb=short