Coverage for security/secret_redactor.py: 87.7%

1"""

2Secret Redactor - Deterministic PII & Secret Detection for Hive Privacy

4Prevents cross-user secret leakage:

5 User A shares an API key in a conversation → it MUST NOT appear in

6 User B's responses via the shared world model.

8Design principle (from project steward):

9 "The hive being hive should not get secrets from one person

10 and reveal to another."

12THREE-LAYER DEFENSE:

13 Layer 1 - SECRET REDACTION (deterministic regex):

14 Pattern-based, zero false negatives on known structured secrets.

15 API keys, tokens, passwords, PEM keys, connection strings.

16 Regex is correct here: structured secrets have known formats.

18 Layer 2 - PER-USER ISOLATION (model-based PII + anonymization):

19 Local LLM (HevolveAI / llama.cpp) semantically detects PII in

20 free text: names, addresses, medical info, financial details.

21 Falls back to regex for emails, phones, URLs, @mentions.

22 user_id/prompt_id/node_id anonymized via SHA-256.

24 Layer 3 - DIFFERENTIAL PRIVACY (statistical noise):

25 Gaussian noise on latency, timestamp quantization, text truncation.

27Why model-based PII (Layer 2)?

28 Regex cannot catch semantic PII in free text: "I live at 123 Oak Lane"

29 or "Dr. Smith diagnosed me with diabetes". The local LLM understands

30 natural language and catches what regex misses.

32Why regex for secrets (Layer 1)?

33 Structured secrets (API keys, JWTs, PEM) have deterministic formats.

34 Regex is faster, auditable, and has zero false negatives on known

35 patterns. No model needed for these.

37Patterns detected:

38 - API keys (OpenAI, AWS, Google, Anthropic, Stripe, etc.)

39 - Generic bearer/auth tokens

40 - Passwords in connection strings or plaintext assignments

41 - Credit card numbers (Luhn-validated)

42 - Private keys (PEM format)

43 - SSH private keys

44 - JWT tokens

45 - AWS access key IDs + secret keys

46 - Connection strings (database, Redis, MongoDB)

47 - Email + password combos

48 - Generic hex/base64 secrets (64+ char)

49"""

51import json

52import os

53import re

54import logging

55import time

56from typing import List, Tuple

58logger = logging.getLogger('hevolve_social')

60# Model-based PII detection - retry cooldown after failure

61_model_last_failure = 0.0

62_MODEL_RETRY_INTERVAL = 60 # Don't retry model for 60s after failure

64# ─── Pattern definitions ─────────────────────────────────────────

66# Each pattern: (name, compiled_regex, replacement_tag)

67_SECRET_PATTERNS: List[Tuple[str, 're.Pattern', str]] = []

70def _add(name: str, pattern: str, tag: str = None):

71 _SECRET_PATTERNS.append((

72 name,

73 re.compile(pattern, re.IGNORECASE | re.DOTALL),

74 tag or f'[REDACTED:{name}]',

75 ))

78# ── API Keys (vendor-specific) ──

79_add('openai_key', r'\bsk-[A-Za-z0-9]{20,}T3BlbkFJ[A-Za-z0-9]{20,}\b')

80_add('openai_key_v2', r'\bsk-(?:proj-)?[A-Za-z0-9_-]{40,}\b')

81_add('anthropic_key', r'\bsk-ant-[A-Za-z0-9_-]{40,}\b')

82_add('aws_access_key', r'\bAKIA[0-9A-Z]{16}\b')

83_add('aws_secret_key', r'(?:aws_secret_access_key|secret_key)\s*[=:]\s*["\']?([A-Za-z0-9/+=]{40})["\']?')

84_add('google_api_key', r'\bAIza[A-Za-z0-9_-]{35}\b')

85_add('stripe_key', r'\b[sr]k_(?:live|test)_[A-Za-z0-9]{24,}\b')

86_add('github_token', r'\b(?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9]{36,}\b')

87_add('slack_token', r'\bxox[baprs]-[A-Za-z0-9-]{10,}\b')

88_add('discord_token', r'[MN][A-Za-z\d]{23,}\.[\w-]{6}\.[\w-]{27,}')

89_add('twilio_key', r'\bSK[a-f0-9]{32}\b')

90_add('sendgrid_key', r'\bSG\.[A-Za-z0-9_-]{22}\.[A-Za-z0-9_-]{43}\b')

91_add('mailgun_key', r'\bkey-[A-Za-z0-9]{32}\b')

92_add('heroku_key', r'[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}')

94# ── Generic bearer/auth tokens ──

95_add('bearer_token',

96 r'(?:bearer|authorization|token|auth)[\s:=]+["\']?([A-Za-z0-9._~+/=-]{32,})["\']?')

98# ── Passwords ──

99_add('password_assignment',

101 r'\s*[=:]\s*["\']([^"\']{8,})["\']')

102_add('password_plaintext',

103 r'(?:password|passwd|pwd)\s*[=:]\s*(\S{8,})')

104

105# ── PEM private keys ──

106_add('pem_private_key',

107 r'-----BEGIN (?:RSA |EC |DSA |OPENSSH )?PRIVATE KEY-----'

108 r'[\s\S]*?'

109 r'-----END (?:RSA |EC |DSA |OPENSSH )?PRIVATE KEY-----')

110

111# ── SSH private keys ──

112_add('ssh_private_key',

113 r'-----BEGIN OPENSSH PRIVATE KEY-----[\s\S]*?-----END OPENSSH PRIVATE KEY-----')

114

115# ── JWT tokens ──

116_add('jwt_token',

117 r'\beyJ[A-Za-z0-9_-]{10,}\.eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b')

118

119# ── Connection strings ──

120_add('connection_string',

122 r'://[^\s"\'<>]{10,}')

123

124# ── Credit card numbers (basic format - 4 groups of 4 digits) ──

125_add('credit_card',

126 r'\b(?:\d{4}[-\s]?){3}\d{4}\b')

127

128# ── Generic long hex secrets (64+ chars, likely SHA/HMAC) ──

129_add('hex_secret',

130 r'(?:secret|key|token|hash|signature)\s*[=:]\s*["\']?([0-9a-fA-F]{64,})["\']?')

131

132# ── Generic long base64 secrets (48+ chars) ──

133_add('base64_secret',

134 r'(?:secret|key|token|private)\s*[=:]\s*["\']?([A-Za-z0-9+/]{48,}={0,3})["\']?')

135

136

137# ─── Luhn check for credit cards ─────────────────────────────────

138

139def _luhn_check(number_str: str) -> bool:

140 """Validate credit card number using Luhn algorithm."""

141 digits = [int(d) for d in number_str if d.isdigit()]

142 if len(digits) < 13 or len(digits) > 19:

143 return False

144 checksum = 0

145 for i, d in enumerate(reversed(digits)):

146 if i % 2 == 1:

147 d *= 2

148 if d > 9:

149 d -= 9

150 checksum += d

151 return checksum % 10 == 0

152

153

154# ─── Main redaction function ─────────────────────────────────────

155

156def redact_secrets(text: str) -> Tuple[str, int]:

157 """Scan text and replace detected secrets with [REDACTED:<type>] tokens.

158

159 Returns:

160 (redacted_text, count_of_redactions)

161

162 Thread-safe: uses only compiled regex (immutable) and local variables.

163 """

164 if not text:

165 return text, 0

166

167 count = 0

168 result = text

169

170 for name, pattern, tag in _SECRET_PATTERNS:

171 matches = pattern.findall(result)

172 if matches:

173 # For credit cards, validate with Luhn before redacting

174 if name == 'credit_card':

175 for match in matches:

176 digits_only = ''.join(c for c in match if c.isdigit())

177 if _luhn_check(digits_only):

178 result = result.replace(match, tag)

179 count += 1

180 else:

181 new_result = pattern.sub(tag, result)

182 if new_result != result:

183 count += len(matches)

184 result = new_result

185

186 return result, count

187

188

189def redact_experience(experience: dict) -> dict:

190 """Full privacy pipeline for shared world model ingestion.

191

192 THREE LAYERS applied in sequence:

193

194 Layer 1 - SECRET REDACTION (deterministic regex):

195 API keys, tokens, passwords, PEM keys, etc. → [REDACTED:<type>]

196

197 Layer 2 - PER-USER ISOLATION:

198 - user_id → anonymized hash (not reversible)

199 - prompt_id → anonymized hash (not reversible)

200 - Quoted text stripped (verbatim content from other systems)

201 - Email addresses stripped

202 - Phone numbers stripped

203 - Names/handles removed from text body

204

205 Layer 3 - DIFFERENTIAL PRIVACY (statistical noise):

206 - Gaussian noise on latency_ms (ε=1.0)

207 - Timestamp quantized to 5-minute buckets (k-anonymity)

208 - node_id anonymized (learn patterns, not node identity)

209 - Text truncated to 500 chars (reduce memorization surface)

210

211 Returns a NEW dict (does not mutate input).

212 """

213 import hashlib

214 import random

215

216 redacted = dict(experience)

217

218 # ── Layer 1: Secret redaction ──

219 total_redactions = 0

220 for field in ('prompt', 'response'):

221 if field in redacted and redacted[field]:

222 redacted[field], n = redact_secrets(str(redacted[field]))

223 total_redactions += n

224

225 if total_redactions > 0:

226 logger.info(

227 f"[SecretRedactor] Redacted {total_redactions} secret(s) "

228 f"from experience (prompt_id={redacted.get('prompt_id', '?')})")

229

230 # ── Layer 2: Per-user isolation ──

231

232 # Anonymize user_id - the world model learns PATTERNS, not who said what

233 if 'user_id' in redacted and redacted['user_id']:

234 uid_hash = hashlib.sha256(

235 str(redacted['user_id']).encode()).hexdigest()[:8]

236 redacted['user_id'] = f'anon_{uid_hash}'

237

238 # Anonymize prompt_id - prevent cross-session correlation

239 if 'prompt_id' in redacted and redacted['prompt_id']:

240 pid_hash = hashlib.sha256(

241 str(redacted['prompt_id']).encode()).hexdigest()[:8]

242 redacted['prompt_id'] = f'prompt_{pid_hash}'

243

244 # Strip PII from text fields - model-based detection with regex fallback.

245 # The model catches semantic PII (names, addresses, medical info) that

246 # regex cannot. Falls back to regex-only if model is unavailable.

247 for field in ('prompt', 'response'):

248 if field in redacted and redacted[field]:

249 redacted[field] = _model_detect_pii(str(redacted[field]))

250

251 # ── Layer 3: Differential privacy ──

252

253 # Gaussian noise on latency (ε ≈ 1.0, sensitivity = 100ms)

254 if 'latency_ms' in redacted and redacted['latency_ms']:

255 noise = random.gauss(0, 50) # σ=50ms

256 redacted['latency_ms'] = max(0, round(redacted['latency_ms'] + noise, 1))

257

258 # Quantize timestamp to 5-minute buckets (k-anonymity)

259 if 'timestamp' in redacted and redacted['timestamp']:

260 bucket = 300 # 5 minutes

261 redacted['timestamp'] = (redacted['timestamp'] // bucket) * bucket

262

263 # Anonymize node_id - learn compute patterns, not node identity

264 if 'node_id' in redacted and redacted['node_id']:

265 nid_hash = hashlib.sha256(

266 str(redacted['node_id']).encode()).hexdigest()[:8]

267 redacted['node_id'] = f'node_{nid_hash}'

268

269 # Truncate text for shared learning - reduces memorization surface.

270 # The world model needs PATTERNS, not full conversations.

271 for field in ('prompt', 'response'):

272 if field in redacted and redacted[field]:

273 redacted[field] = redacted[field][:500]

274

275 return redacted

276

277

278# ─── Layer 2: PII stripping patterns ────────────────────────────

279

280# Email addresses

281_EMAIL_PATTERN = re.compile(

282 r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b')

283

284# Phone numbers (various formats)

285_PHONE_PATTERN = re.compile(

286 r'(?:\+?\d{1,3}[-.\s]?)?\(?\d{2,4}\)?[-.\s]?\d{3,4}[-.\s]?\d{3,4}\b')

287

288# Quoted text (content from other systems/people - high risk of verbatim leak)

289_QUOTED_PATTERN = re.compile(

290 r'(?:^|\n)\s*>.*(?:\n\s*>.*)*', re.MULTILINE)

291

292# URLs with query params (may contain tokens/session IDs)

293_URL_WITH_PARAMS = re.compile(

294 r'https?://\S+[?&]\S+')

295

296# @mentions / handles

297_MENTION_PATTERN = re.compile(r'@[A-Za-z0-9_]{2,30}\b')

298

299

300def _strip_pii(text: str) -> str:

301 """Strip PII patterns from text for per-user isolation.

302

303 Removes emails, phone numbers, quoted content, URL params, @mentions.

304 Deterministic and fast (regex-only).

305 """

306 if not text:

307 return text

308

309 result = _EMAIL_PATTERN.sub('[EMAIL]', text)

310 result = _PHONE_PATTERN.sub('[PHONE]', result)

311 result = _QUOTED_PATTERN.sub('\n[QUOTED_CONTENT_REMOVED]', result)

312 result = _URL_WITH_PARAMS.sub('[URL_REDACTED]', result)

313 result = _MENTION_PATTERN.sub('[HANDLE]', result)

314

315 return result

316

317

318def _model_detect_pii(text: str) -> str:

319 """Use local LLM to semantically detect PII in free text.

320

321 Catches PII that regex cannot: names, addresses, medical info,

322 financial details, biographical information.

323

324 Applies regex FIRST (fast, catches emails/phones/URLs), then

325 enhances with model-detected entities (names, addresses, etc.).

326

327 Falls back to regex-only if model is unavailable.

328 """

329 global _model_last_failure

330

331 if not text or len(text) < 20:

332 return _strip_pii(text)

333

334 # Skip model call if recently failed (avoid repeated timeouts)

335 if time.time() - _model_last_failure < _MODEL_RETRY_INTERVAL:

336 return _strip_pii(text)

337

338 # Apply regex first (always runs, sub-ms)

339 regex_result = _strip_pii(text)

340

341 try:

342 import requests as _req

343

344 # Use whatever local LLM endpoint is available

345 llm_url = os.environ.get(

346 'HEVOLVE_LOCAL_LLM_URL',

347 os.environ.get('HEVOLVEAI_API_URL', 'http://localhost:8080')

348 )

349

350 resp = _req.post(

351 f'{llm_url.rstrip("/")}/v1/chat/completions',

352 json={

353 'model': 'local',

354 'messages': [{

355 'role': 'user',

356 'content': (

357 "Extract ALL personally identifiable information (PII) "

358 "from the text below. PII includes: full names, "

359 "street addresses, dates of birth, government IDs "

360 "(SSN, passport), medical conditions/diagnoses, "

361 "financial account numbers, IP addresses, "

362 "biometric data, and any other info that could "

363 "identify a specific person.\n"

364 "Do NOT include generic words, technical terms, "

365 "or non-identifying information.\n"

366 "Return ONLY a JSON array of the exact PII strings "

367 "found. Return [] if none found.\n\n"

368 f"Text: {text[:1000]}\n\nPII:"

369 ),

370 }],

371 'max_tokens': 256,

372 'temperature': 0.0,

373 },

374 timeout=3,

375 )

376

377 if resp.status_code == 200:

378 result = resp.json()

379 content = result.get('choices', [{}])[0].get(

380 'message', {}).get('content', '')

381

382 # Parse JSON array from response

383 start = content.find('[')

384 end = content.rfind(']')

385 if start != -1 and end != -1:

386 pii_items = json.loads(content[start:end + 1])

387 if isinstance(pii_items, list):

388 for item in pii_items:

389 if isinstance(item, str) and len(item) >= 2:

390 regex_result = regex_result.replace(

391 item, '[PII_REDACTED]')

392 return regex_result

393 # Non-200 or unparseable - regex result is still good

394 _model_last_failure = time.time()

395 return regex_result

396

397 except Exception:

398 _model_last_failure = time.time()

399 return regex_result

400

401

402def contains_secrets(text: str) -> bool:

403 """Quick check: does this text contain any detectable secrets?"""

404 _, count = redact_secrets(text)

405 return count > 0

Coverage for security / secret_redactor.py: 87.7%

138 statements