fix(links): redact credential-shaped opts by pattern, not by allowlist (CON-877)

common/lib/redaction.py

@@ -0,0 +1,36 @@
+"""Redaction of link options before they leave the process.
+
+Links record their `opts` into `analysis[].vendor_schema` and into logs. Those opts
+carry provider credentials. Each link used to keep its own allowlist of names to drop,
+which fails the moment a new credential option is added: a provider key reached every
+transcribed vCon that way. Match by pattern instead, so an unknown credential is dropped
+by default.
+"""
+
+from typing import Any, Dict
+
+# ponytail: substring match on the name. Cheap, and it fails closed on new options.
+SENSITIVE_NAME_PARTS = (
+    "key",
+    "token",
+    "secret",
+    "password",
+    "passwd",
+    "credential",
+    "proxy_url",
+)
+
+
+# Not credentials, but internal endpoints that must not be recorded into stored vCons.
+# `policy_url` and the like are public links and stay.
+SENSITIVE_NAMES = {"send_ai_usage_data_to_url"}
+
+
+def is_sensitive_name(name: str) -> bool:
+    lowered = (name or "").lower()
+    return lowered in SENSITIVE_NAMES or any(part in lowered for part in SENSITIVE_NAME_PARTS)
+
+
+def safe_opts(opts: Dict[str, Any]) -> Dict[str, Any]:
+    """Return opts with every credential-shaped option removed."""
+    return {k: v for k, v in (opts or {}).items() if not is_sensitive_name(k)}

common/tests/test_redaction.py

@@ -0,0 +1,30 @@
+from lib.redaction import safe_opts
+
+
+def test_drops_credential_shaped_options():
+    opts = {
+        "model": "distil-whisper/distil-large-v2",
+        "language": "en",
+        "OPENAI_API_KEY": "sk-live",
+        "LITELLM_MASTER_KEY": "sk-litellm",
+        "LITELLM_PROXY_URL": "https://proxy.invalid",
+        "ai_usage_api_token": "t",
+        "aws_secret_access_key": "s",
+        "db_password": "p",
+        "send_ai_usage_data_to_url": "https://usage.internal",
+    }
+    assert safe_opts(opts) == {"model": "distil-whisper/distil-large-v2", "language": "en"}
+
+
+def test_unknown_credential_option_is_dropped_too():
+    # the bug: an allowlist misses the next key that gets added
+    assert safe_opts({"SOME_NEW_PROVIDER_KEY": "x"}) == {}
+
+
+def test_public_urls_are_kept():
+    assert safe_opts({"policy_url": "https://example.com/dpa"}) == {"policy_url": "https://example.com/dpa"}
+
+
+def test_handles_empty():
+    assert safe_opts({}) == {}
+    assert safe_opts(None) == {}

conserver/links/analyze/__init__.py

@@ -1,3 +1,4 @@
+from lib.redaction import safe_opts
 from lib.vcon_redis import VconRedis
 from lib.logging_utils import init_logger
 from lib.openai_client import get_openai_client, get_vendor_from_opts
@@ -142,13 +143,7 @@ def run(
             continue
 
         # Filter out sensitive keys from logging
-        filtered_opts = {
-            k: v for k, v in opts.items()
-            if k not in (
-                "OPENAI_API_KEY", "AZURE_OPENAI_API_KEY",
-                "AZURE_OPENAI_ENDPOINT", "ai_usage_api_token"
-            )
-        }
+        filtered_opts = safe_opts(opts)
         logger.info(
             "Analysing dialog %s with options: %s",
             index,

conserver/links/analyze_and_label/__init__.py

@@ -1,3 +1,4 @@
+from lib.redaction import safe_opts
 from lib.vcon_redis import VconRedis
 from lib.logging_utils import init_logger
 from lib.openai_client import get_openai_client
@@ -121,7 +122,7 @@ def run(
         logger.info(
             "Analysing dialog %s with options: %s",
             index,
-            {k: v for k, v in opts.items() if k != "OPENAI_API_KEY"},
+            safe_opts(opts),
         )
         start = time.time()
         try:

conserver/links/analyze_vcon/__init__.py

@@ -1,3 +1,4 @@
+from lib.redaction import safe_opts
 from lib.vcon_redis import VconRedis
 from lib.logging_utils import init_logger
 from lib.openai_client import get_openai_client
@@ -135,7 +136,7 @@ def run(
 
     logger.info(
         "Analyzing entire vCon with options: %s",
-        {k: v for k, v in opts.items() if k != "OPENAI_API_KEY"},
+        safe_opts(opts),
     )
 
     start = time.time()

conserver/links/check_and_tag/__init__.py

@@ -1,3 +1,4 @@
+from lib.redaction import safe_opts
 from lib.vcon_redis import VconRedis
 from lib.logging_utils import init_logger
 from lib.openai_client import get_openai_client
@@ -151,7 +152,7 @@ def run(
         logger.info(
             "Analysing dialog %s with options: %s",
             index,
-            {k: v for k, v in opts.items() if k != "OPENAI_API_KEY"},
+            safe_opts(opts),
         )
         start = time.time()
         try:

conserver/links/deepgram_link/__init__.py

@@ -5,6 +5,7 @@
 ``links.transcribe`` dispatcher and emits a one-time deprecation warning.
 """
 
+from lib.redaction import safe_opts
 from typing import Optional
 import os
 import tempfile
@@ -296,14 +297,7 @@ def run(
 
         # Prepare vendor schema, omitting credentials
         vendor_schema = {}
-        sensitive_keys = {
-            "DEEPGRAM_KEY",
-            "ai_usage_api_token",
-            "send_ai_usage_data_to_url",
-            "LITELLM_PROXY_URL",
-            "LITELLM_MASTER_KEY",
-        }
-        vendor_schema["opts"] = {k: v for k, v in opts.items() if k not in sensitive_keys}
+        vendor_schema["opts"] = safe_opts(opts)
 
         # Add the transcript analysis to the vCon
         vCon.add_analysis(

conserver/links/detect_engagement/__init__.py

@@ -1,3 +1,4 @@
+from lib.redaction import safe_opts
 from lib.vcon_redis import VconRedis
 from lib.logging_utils import init_logger
 from lib.openai_client import get_openai_client
@@ -119,7 +120,7 @@ def run(
         logger.info(
             "Analyzing engagement for dialog %s with options: %s",
             index,
-            {k: v for k, v in opts.items() if k != "OPENAI_API_KEY"},
+            safe_opts(opts),
         )
         start = time.time()
         try:

conserver/links/groq_whisper/__init__.py

@@ -11,6 +11,7 @@
 transcription results.
 """
 
+from lib.redaction import safe_opts
 import base64
 import hashlib
 import logging
@@ -413,10 +414,7 @@ def run(
 
         # Prepare vendor schema without sensitive data
         vendor_schema = {
-            "opts": {
-                k: v
-                for k, v in opts.items() if k != "API_KEY"
-            }
+            "opts": safe_opts(opts)
         }
 
         # Add transcription analysis to vCon

conserver/links/hugging_face_whisper/__init__.py

@@ -10,6 +10,7 @@
 transcription results.
 """
 
+from lib.redaction import safe_opts
 import base64
 import hashlib
 import logging
@@ -219,7 +220,7 @@ def run(
         logger.info(result)
 
         # Prepare vendor schema without sensitive data
-        vendor_schema = {"opts": {k: v for k, v in opts.items() if k != "API_KEY"}}
+        vendor_schema = {"opts": safe_opts(opts)}
 
         # Add transcription analysis to vCon
         vCon.add_analysis(

conserver/links/openai_transcribe/__init__.py

@@ -6,6 +6,7 @@
 emits a one-time deprecation warning.
 """
 
+from lib.redaction import safe_opts
 import re
 from urllib.parse import unquote, urlparse
 from lib.logging_utils import init_logger
@@ -575,8 +576,7 @@ def run(
 
         # Prepare vendor schema, omitting credentials
         vendor_schema = {}
-        sensitive_keys = {"OPENAI_API_KEY", "AZURE_OPENAI_API_KEY", "ai_usage_api_token", "send_ai_usage_data_to_url"}
-        vendor_schema["opts"] = {k: v for k, v in opts.items() if k not in sensitive_keys}
+        vendor_schema["opts"] = safe_opts(opts)
 
         # Add the transcript analysis to the vCon
         vCon.add_analysis(

conserver/links/openai_transcribe/tests/test_openai_transcribe_unit.py

@@ -301,6 +301,7 @@ def test_run_skips_irrelevant_dialogs_and_redacts_sensitive_opts():
                 "minimum_duration": 3,
                 "model": "gpt-4o-transcribe",
                 "OPENAI_API_KEY": "secret",
+                "LITELLM_MASTER_KEY": "litellm-secret",
                 "ai_usage_api_token": "usage-secret",
                 "send_ai_usage_data_to_url": "https://usage.example",
             },
@@ -319,6 +320,7 @@ def test_run_skips_irrelevant_dialogs_and_redacts_sensitive_opts():
     assert redacted_opts["use_silence_chunking"] is True
     assert "OPENAI_API_KEY" not in redacted_opts
     assert "AZURE_OPENAI_API_KEY" not in redacted_opts
+    assert "LITELLM_MASTER_KEY" not in redacted_opts
     assert "ai_usage_api_token" not in redacted_opts
     assert "send_ai_usage_data_to_url" not in redacted_opts
     redis_client.store_vcon.assert_called_once_with(fake_vcon)