"""Catch unsafe output and comparison-boundary bugs that end-to-end timings cannot force.""" import base64 import json from pathlib import Path import tempfile import unittest from compare import compare from safety import SecretScanner class PerformanceContract(unittest.TestCase): def result(self, value, noise=0, mode="replay"): return { "commit": "candidate", "mode": mode, "gated": {"cost": {"value": value, "noise": noise}}, } def test_baseline_noise_floor_and_absolute_budget_all_apply(self): budgets = {"metrics": {"cost": {"ceiling": 200, "floor": 10, "kind": "memory"}}} baseline = self.result(100, 5) baseline["commit"] = "baseline" self.assertEqual( compare(self.result(115), baseline, budgets)["verdict"], "pass" ) self.assertEqual( compare(self.result(116), baseline, budgets)["gated"]["cost"]["verdict"], "regressed", ) self.assertEqual( compare(self.result(201, 100), baseline, budgets)["gated"]["cost"][ "verdict" ], "over_budget", ) self.assertEqual( compare(self.result(201, mode="live"), baseline, budgets)["verdict"], "pass" ) def test_byte_increase_is_not_hidden_by_memory_noise(self): budgets = { "metrics": {"cost": {"ceiling": 2000, "floor": 1000, "kind": "bytes"}} } self.assertEqual( compare(self.result(1011, 999), self.result(1000, 999), budgets)["verdict"], "fail", ) def test_rejects_credentials_split_across_sse_deltas_without_replacing_output(self): secret = "private-value-for-this-test" scanner = SecretScanner(extra=[secret]) chunks = [secret[:11], secret[11:]] body = "".join( "data: " + json.dumps({"choices": [{"delta": {"content": chunk}}]}) + "\n\n" for chunk in chunks ) recording = {"status": 200, "body": base64.b64encode(body.encode()).decode()} with tempfile.TemporaryDirectory() as directory: path = Path(directory) / "output.json" path.write_text("previous output") with self.assertRaisesRegex(ValueError, "credential value"): scanner.write_json(path, recording) self.assertEqual(path.read_text(), "previous output") def test_rejects_encoded_credentials_and_key_shapes(self): scanner = SecretScanner(extra=["test-secret-value"]) for content in ("test%2Dsecret%2Dvalue", "sk-" + "x" * 32): with self.assertRaises(ValueError): scanner.check(content) if __name__ == "__main__": unittest.main()