railwaysystem/scripts/qa/real_secret_scanner.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Real Secret Scanner across all git tracked files
"""
import subprocess
import re
import os

EXTENSIONS = (
    '.java', '.xml', '.yml', '.yaml', '.properties',
    '.js', '.vue', '.py', '.sh', '.md', 'package.json'
)

# 真实真实凭证特征 (根据用户在前序轮次中提到的真实特征或已知真实值)
# 任何包含这些特征的为 REAL_SECRET
KNOWN_REAL_SECRET_PATTERNS = [
    re.compile(r'sm15465677241_gh'),
    re.compile(r'fcd1cf253af502ee59f9e5d154e008cd'),
    re.compile(r'BEGIN (?:RSA|EC|OPENSSH|DSA|PGP) PRIVATE KEY')
]

# 通用密码/敏感特征检测
PATTERNS = {
    "DB_PASSWORD": [
        re.compile(r'password\s*[:=]\s*([^\s#"\',]+)', re.IGNORECASE),
        re.compile(r'spring\.datasource\.druid\.master\.password\s*[:=]\s*([^\s#"\',]+)', re.IGNORECASE)
    ],
    "QA_PASSWORD": [
        re.compile(r'QA_ADMIN_PASSWORD\s*=\s*["\x27]([^"\x27]+)["\x27]'),
        re.compile(r'env\.get\(["\x27]QA_ADMIN_PASSWORD["\x27]\s*,\s*["\x27]([^"\x27]+)["\x27]\)')
    ],
    "IDENTITY_SECRET": [
        re.compile(r'TRAIN_STUDENT_IDENTITY_ENCRYPTION_KEY\s*=\s*["\x27]([^"\x27]+)["\x27]'),
        re.compile(r'TRAIN_STUDENT_IDENTITY_HASH_KEY\s*=\s*["\x27]([^"\x27]+)["\x27]'),
        re.compile(r'identity\.encryption\.key\s*[:=]\s*([^\s#"\',]+)', re.IGNORECASE)
    ],
    "REAL_DMS_SECRET": [
        re.compile(r'DMS_APP_SECRET\s*=\s*["\x27]([^"\x27]+)["\x27]'),
        re.compile(r'dms\.appSecret\s*[:=]\s*([^\s#"\',]+)', re.IGNORECASE)
    ],
    "TOKEN_SECRET": [
        re.compile(r'token\.secret\s*[:=]\s*([^\s#"\',]+)', re.IGNORECASE),
        re.compile(r'jwt\.secret\s*[:=]\s*([^\s#"\',]+)', re.IGNORECASE)
    ]
}

# 允许的占位符或环境变量名或已知测试固定值
SAFE_PLACEHOLDERS = {
    '${spring.datasource.druid.master.password}',
    '${dms.appsecret:}',
    '${token.secret:}',
    'admin123', # RuoYi 原版框架示例测试密码
    '123456',
    '[redacted]',
    'null',
    'none',
    ''
}

def scan():
    files = subprocess.check_output(['git', 'ls-files']).decode('utf-8').splitlines()
    target_files = [f for f in files if f.endswith(EXTENSIONS) or os.path.basename(f) == 'package.json']
    
    findings = []
    counts = {
        "TRACKED_QA_PASSWORD_COUNT": 0,
        "TRACKED_DB_SECRET_COUNT": 0,
        "TRACKED_IDENTITY_SECRET_COUNT": 0,
        "TRACKED_REAL_DMS_SECRET_COUNT": 0,
        "TRACKED_TOKEN_SECRET_COUNT": 0,
        "REAL_SECRET_COMMITTED_COUNT": 0
    }

    for fpath in target_files:
        if not os.path.exists(fpath) or os.path.abspath(fpath) == os.path.abspath(__file__):
            continue
        try:
            with open(fpath, 'r', encoding='utf-8', errors='ignore') as f:
                for idx, line in enumerate(f, 1):
                    # 1. 检查已知真实凭证
                    for real_pat in KNOWN_REAL_SECRET_PATTERNS:
                        if real_pat.search(line):
                            if 'node_modules' in fpath or 'test/' in fpath or 'fixtures' in fpath:
                                findings.append((fpath, idx, "TEST_FIXTURE_KEY", "[REDACTED]"))
                            else:
                                counts["REAL_SECRET_COMMITTED_COUNT"] += 1
                                findings.append((fpath, idx, "REAL_COMMITTED_SECRET", "[REDACTED]"))

                    # 2. 检查各类敏感配置
                    for sec_type, regex_list in PATTERNS.items():
                        for reg in regex_list:
                            m = reg.search(line)
                            if m:
                                val = m.group(1).strip()
                                # 判断是否是真实 secret 还是 placeholder
                                if val.startswith('${') or val.startswith('"$') or val.startswith("'$") or val.startswith('$') or val.lower() in SAFE_PLACEHOLDERS or 'getenv' in line.lower() or 'os.environ' in line.lower():
                                    # 属于环境变量或占位符
                                    continue
                                else:
                                    # 记录命中的类别
                                    counts[f"TRACKED_{sec_type}_COUNT"] += 1
                                    findings.append((fpath, idx, sec_type, "[REDACTED]"))
        except Exception:
            pass

    return counts, findings

if __name__ == '__main__':
    counts, findings = scan()
    print("================ REAL SECRET AUDIT REPORT ================")
    print(f"TRACKED_QA_PASSWORD_COUNT = {counts['TRACKED_QA_PASSWORD_COUNT']}")
    print(f"TRACKED_DB_SECRET_COUNT = {counts['TRACKED_DB_SECRET_COUNT']}")
    print(f"TRACKED_IDENTITY_SECRET_COUNT = {counts['TRACKED_IDENTITY_SECRET_COUNT']}")
    print(f"TRACKED_REAL_DMS_SECRET_COUNT = {counts['TRACKED_REAL_DMS_SECRET_COUNT']}")
    print(f"TRACKED_TOKEN_SECRET_COUNT = {counts['TRACKED_TOKEN_SECRET_COUNT']}")
    print(f"REAL_SECRET_COMMITTED_COUNT = {counts['REAL_SECRET_COMMITTED_COUNT']}")
    print(f"TOTAL FINDINGS = {len(findings)}")
    print("----------------------------------------------------------")
    for fpath, line_no, stype, red in findings[:30]:
        print(f"{fpath}:{line_no} [{stype}] {red}")