feat: dlp scanner — multi-format data loss prevention scanner
Scans files, databases, and network captures for sensitive data (PII, credentials, financial, health) with 30+ detection rules, protocol-aware network analysis, DNS exfiltration detection, compliance mapping (HIPAA, PCI-DSS, GDPR, CCPA), and multi-format reporting (console, JSON, SARIF, CSV).
This commit is contained in:
CarterPerez-dev
parent
f95785ac8d
commit
8d5a177bca
|
|
@ -0,0 +1,86 @@
|
|||
# ©AngelaMos | 2026
|
||||
# .dlp-scanner.yml
|
||||
|
||||
scan:
|
||||
file:
|
||||
max_file_size_mb: 100
|
||||
recursive: true
|
||||
exclude_patterns:
|
||||
- "*.pyc"
|
||||
- "__pycache__"
|
||||
- ".git"
|
||||
- "node_modules"
|
||||
- ".venv"
|
||||
include_extensions:
|
||||
- ".pdf"
|
||||
- ".docx"
|
||||
- ".xlsx"
|
||||
- ".xls"
|
||||
- ".csv"
|
||||
- ".json"
|
||||
- ".xml"
|
||||
- ".yaml"
|
||||
- ".yml"
|
||||
- ".txt"
|
||||
- ".log"
|
||||
- ".eml"
|
||||
- ".msg"
|
||||
- ".parquet"
|
||||
- ".avro"
|
||||
- ".tar.gz"
|
||||
- ".tar.bz2"
|
||||
- ".zip"
|
||||
|
||||
database:
|
||||
sample_percentage: 5
|
||||
max_rows_per_table: 10000
|
||||
timeout_seconds: 30
|
||||
exclude_tables: []
|
||||
include_tables: []
|
||||
|
||||
network:
|
||||
bpf_filter: ""
|
||||
entropy_threshold: 7.2
|
||||
dns_label_entropy_threshold: 4.0
|
||||
max_packets: 0
|
||||
|
||||
detection:
|
||||
min_confidence: 0.20
|
||||
severity_threshold: "low"
|
||||
context_window_tokens: 10
|
||||
enable_rules:
|
||||
- "*"
|
||||
disable_rules: []
|
||||
allowlists:
|
||||
values:
|
||||
- "123-45-6789"
|
||||
- "000-00-0000"
|
||||
- "4111111111111111"
|
||||
domains:
|
||||
- "example.com"
|
||||
- "test.com"
|
||||
file_patterns:
|
||||
- "test_*"
|
||||
- "*_fixture*"
|
||||
- "mock_*"
|
||||
|
||||
compliance:
|
||||
frameworks:
|
||||
- "HIPAA"
|
||||
- "PCI_DSS"
|
||||
- "GDPR"
|
||||
- "CCPA"
|
||||
- "SOX"
|
||||
- "GLBA"
|
||||
|
||||
output:
|
||||
format: "console"
|
||||
output_file: ""
|
||||
redaction_style: "partial"
|
||||
verbose: false
|
||||
color: true
|
||||
|
||||
logging:
|
||||
level: "INFO"
|
||||
json_output: false
|
||||
log_file: ""
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
# ©AngelaMos | 2026
|
||||
# .env.example
|
||||
|
||||
# PostgreSQL
|
||||
PGHOST=localhost
|
||||
PGPORT=5432
|
||||
PGUSER=dlp_scanner
|
||||
PGPASSWORD=changeme
|
||||
PGDATABASE=target_db
|
||||
|
||||
# MySQL
|
||||
MYSQL_HOST=localhost
|
||||
MYSQL_PORT=3306
|
||||
MYSQL_USER=dlp_scanner
|
||||
MYSQL_PASSWORD=changeme
|
||||
MYSQL_DATABASE=target_db
|
||||
|
||||
# MongoDB
|
||||
MONGO_URI=mongodb://localhost:27017
|
||||
MONGO_DATABASE=target_db
|
||||
|
||||
# Logging
|
||||
DLP_LOG_LEVEL=INFO
|
||||
DLP_LOG_JSON=false
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
docs/
|
||||
__pycache__/
|
||||
*.pyc
|
||||
.env
|
||||
.venv/
|
||||
*.egg-info/
|
||||
dist/
|
||||
build/
|
||||
.mypy_cache/
|
||||
.ruff_cache/
|
||||
.pytest_cache/
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
[style]
|
||||
based_on_style = pep8
|
||||
column_limit = 75
|
||||
indent_width = 4
|
||||
continuation_indent_width = 4
|
||||
indent_closing_brackets = false
|
||||
dedent_closing_brackets = true
|
||||
indent_blank_lines = false
|
||||
spaces_before_comment = 2
|
||||
spaces_around_power_operator = false
|
||||
spaces_around_default_or_named_assign = true
|
||||
space_between_ending_comma_and_closing_bracket = false
|
||||
space_inside_brackets = false
|
||||
spaces_around_subscript_colon = true
|
||||
blank_line_before_nested_class_or_def = false
|
||||
blank_line_before_class_docstring = false
|
||||
blank_lines_around_top_level_definition = 2
|
||||
blank_lines_between_top_level_imports_and_variables = 2
|
||||
blank_line_before_module_docstring = false
|
||||
split_before_logical_operator = true
|
||||
split_before_first_argument = true
|
||||
split_before_named_assigns = true
|
||||
split_complex_comprehension = true
|
||||
split_before_expression_after_opening_paren = false
|
||||
split_before_closing_bracket = true
|
||||
split_all_comma_separated_values = true
|
||||
split_all_top_level_comma_separated_values = false
|
||||
coalesce_brackets = false
|
||||
each_dict_entry_on_separate_line = true
|
||||
allow_multiline_lambdas = false
|
||||
allow_multiline_dictionary_keys = false
|
||||
split_penalty_import_names = 0
|
||||
join_multiple_lines = false
|
||||
align_closing_bracket_with_visual_indent = true
|
||||
arithmetic_precedence_indication = false
|
||||
split_penalty_for_added_line_split = 275
|
||||
use_tabs = false
|
||||
split_before_dot = false
|
||||
split_arguments_when_comma_terminated = true
|
||||
i18n_function_call = ['_', 'N_', 'gettext', 'ngettext']
|
||||
i18n_comment = ['# Translators:', '# i18n:']
|
||||
split_penalty_comprehension = 80
|
||||
split_penalty_after_opening_bracket = 280
|
||||
split_penalty_before_if_expr = 0
|
||||
split_penalty_bitwise_operator = 290
|
||||
split_penalty_logical_operator = 0
|
||||
|
|
@ -0,0 +1,112 @@
|
|||
```ruby
|
||||
██████╗ ██╗ ██████╗ ███████╗ ██████╗ █████╗ ███╗ ██╗
|
||||
██╔══██╗██║ ██╔══██╗ ██╔════╝██╔════╝██╔══██╗████╗ ██║
|
||||
██║ ██║██║ ██████╔╝█████╗███████╗██║ ███████║██╔██╗ ██║
|
||||
██║ ██║██║ ██╔═══╝ ╚════╝╚════██║██║ ██╔══██║██║╚██╗██║
|
||||
██████╔╝███████╗██║ ███████║╚██████╗██║ ██║██║ ╚████║
|
||||
╚═════╝ ╚══════╝╚═╝ ╚══════╝ ╚═════╝╚═╝ ╚═╝╚═╝ ╚═══╝
|
||||
```
|
||||
|
||||
[](https://github.com/CarterPerez-dev/Cybersecurity-Projects/tree/main/PROJECTS/intermediate/dlp-scanner)
|
||||
[](https://python.org)
|
||||
[](https://www.gnu.org/licenses/agpl-3.0)
|
||||
|
||||
> Data Loss Prevention scanner for files, databases, and network traffic.
|
||||
|
||||
*This is a quick overview. Security theory, architecture, and full walkthroughs are in the [learn modules](#learn).*
|
||||
|
||||
## What It Does
|
||||
|
||||
- Scans files (PDF, DOCX, XLSX, CSV, JSON, XML, YAML, Parquet, Avro, archives, emails) for PII, credentials, financial data, and PHI
|
||||
- Scans databases (PostgreSQL, MySQL, MongoDB, SQLite) with schema introspection and sampling
|
||||
- Scans network captures (PCAP/PCAPNG) with protocol parsing, TCP reassembly, and DNS exfiltration detection
|
||||
- Confidence scoring pipeline: regex match, checksum validation (Luhn, Mod-97, Mod-11), context keyword proximity, entity co-occurrence
|
||||
- Maps findings to compliance frameworks (HIPAA, PCI-DSS, GDPR, CCPA, SOX, GLBA, FERPA)
|
||||
- Reports in console (Rich tables), JSON, SARIF 2.1.0, or CSV
|
||||
|
||||
## Quick Start
|
||||
|
||||
```bash
|
||||
bash install.sh
|
||||
dlp-scan file ./data
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
```bash
|
||||
dlp-scan file ./data/employees/ # scan a directory
|
||||
dlp-scan file ./report.pdf -f json # scan a file, JSON output
|
||||
dlp-scan db postgres://user:pass@host/db # scan PostgreSQL
|
||||
dlp-scan db sqlite:///path/to/local.db # scan SQLite
|
||||
dlp-scan network capture.pcap # scan network traffic
|
||||
dlp-scan file ./data -f sarif -o results.sarif # SARIF for CI/CD
|
||||
dlp-scan report convert results.json -f csv # convert report format
|
||||
dlp-scan report summary results.json # print summary stats
|
||||
```
|
||||
|
||||
### Global Options
|
||||
|
||||
```
|
||||
--config, -c Path to YAML config file
|
||||
--verbose, -v Enable debug logging
|
||||
--version Show version
|
||||
```
|
||||
|
||||
### Output Formats
|
||||
|
||||
| Format | Flag | Use Case |
|
||||
|--------|------|----------|
|
||||
| Console | `-f console` | Interactive review with Rich tables |
|
||||
| JSON | `-f json` | Structured analysis and archival |
|
||||
| SARIF | `-f sarif` | GitHub code scanning, CI/CD integration |
|
||||
| CSV | `-f csv` | Compliance team export, spreadsheet import |
|
||||
|
||||
## Stack
|
||||
|
||||
**Language:** Python 3.12+
|
||||
|
||||
**CLI:** Typer 0.15+ with Rich integration
|
||||
|
||||
**Detection:** Regex + checksum validators + Shannon entropy + context keyword scoring
|
||||
|
||||
**File Formats:** PyMuPDF, python-docx, openpyxl, xlrd, defusedxml, lxml, pyarrow, fastavro, extract-msg
|
||||
|
||||
**Databases:** asyncpg (PostgreSQL), aiomysql (MySQL), pymongo async (MongoDB), aiosqlite (SQLite)
|
||||
|
||||
**Network:** dpkt (PCAP parsing), TCP reassembly, DPI protocol identification, DNS exfiltration heuristics
|
||||
|
||||
**Config:** Pydantic 2.10+ models with YAML config loading (ruamel.yaml)
|
||||
|
||||
**Quality:** ruff, mypy (strict), yapf, pytest + hypothesis, structlog
|
||||
|
||||
## Configuration
|
||||
|
||||
Copy `.dlp-scanner.yml` to your project root and customize. Key settings:
|
||||
|
||||
```yaml
|
||||
detection:
|
||||
min_confidence: 0.20 # minimum score to report
|
||||
enable_rules: ["*"] # glob patterns for rule IDs
|
||||
allowlists:
|
||||
values: ["123-45-6789"] # suppress known test values
|
||||
|
||||
output:
|
||||
format: "console" # console, json, sarif, csv
|
||||
redaction_style: "partial" # partial, full, none
|
||||
```
|
||||
|
||||
## Learn
|
||||
|
||||
This project includes step-by-step learning materials covering security theory, architecture, and implementation.
|
||||
|
||||
| Module | Topic |
|
||||
|--------|-------|
|
||||
| [00 - Overview](learn/00-OVERVIEW.md) | Prerequisites and quick start |
|
||||
| [01 - Concepts](learn/01-CONCEPTS.md) | DLP theory and real-world breaches |
|
||||
| [02 - Architecture](learn/02-ARCHITECTURE.md) | System design and data flow |
|
||||
| [03 - Implementation](learn/03-IMPLEMENTATION.md) | Code walkthrough |
|
||||
| [04 - Challenges](learn/04-CHALLENGES.md) | Extension ideas and exercises |
|
||||
|
||||
## License
|
||||
|
||||
[AGPLv3](https://www.gnu.org/licenses/agpl-3.0)
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
#!/usr/bin/env bash
|
||||
# ©AngelaMos | 2026
|
||||
# install.sh
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
command -v uv >/dev/null 2>&1 || {
|
||||
echo "Installing uv..."
|
||||
curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
export PATH="$HOME/.local/bin:$PATH"
|
||||
}
|
||||
|
||||
echo "Syncing dependencies..."
|
||||
uv sync
|
||||
|
||||
echo "Downloading spaCy model (optional, for NLP-based detection)..."
|
||||
uv run python -m spacy download en_core_web_sm 2>/dev/null || true
|
||||
|
||||
echo ""
|
||||
echo "Setup complete. Run the scanner with:"
|
||||
echo " uv run dlp-scan --help"
|
||||
echo ""
|
||||
echo "Quick start:"
|
||||
echo " uv run dlp-scan scan file ./path/to/scan"
|
||||
echo " uv run dlp-scan scan db sqlite:///path/to/db.sqlite3"
|
||||
echo " uv run dlp-scan scan network ./capture.pcap"
|
||||
|
|
@ -0,0 +1,76 @@
|
|||
# 00-OVERVIEW.md
|
||||
|
||||
# DLP Scanner
|
||||
|
||||
## What This Is
|
||||
|
||||
A command-line Data Loss Prevention scanner that detects sensitive data across three surfaces: files (PDF, DOCX, XLSX, CSV, JSON, XML, YAML, Parquet, Avro, archives, emails), databases (PostgreSQL, MySQL, MongoDB, SQLite), and network captures (PCAP/PCAPNG with protocol parsing and TCP reassembly). It uses a confidence scoring pipeline combining regex matching, checksum validation (Luhn for credit cards, Mod-97 for IBANs, Mod-11 for NHS numbers), keyword proximity analysis, and Shannon entropy detection. Findings are classified by severity and mapped to compliance frameworks (HIPAA, PCI-DSS, GDPR, CCPA, SOX, GLBA, FERPA). Output supports console Rich tables, JSON, SARIF 2.1.0 for CI/CD, and CSV for compliance teams.
|
||||
|
||||
## Why This Matters
|
||||
|
||||
Data breaches involving PII exposure keep appearing because organizations cannot find sensitive data they do not know exists. The 2017 Equifax breach exposed 147 million SSNs from an unpatched Apache Struts application, but the underlying problem was that SSNs were stored in plaintext across multiple database tables without anyone tracking where that data lived. In 2019, Capital One lost 100 million credit applications from an S3 bucket because a misconfigured WAF allowed server-side request forgery, and nobody had scanned those files to realize unencrypted SSNs and credit card numbers sat in flat CSV exports. The Marriott breach (2018) exposed 500 million records including 5.25 million unencrypted passport numbers, partially because the Starwood reservation system merged without a data inventory that would have flagged those fields as sensitive.
|
||||
|
||||
These are not failure-of-firewall problems. They are failure-of-visibility problems. DLP tools exist to answer "where is our sensitive data?" before attackers answer it for you. Commercial solutions (Symantec DLP, Microsoft Purview, Netskope) cost six figures and require enterprise deployment, but the core detection logic is straightforward: pattern matching with validation, context analysis to reduce false positives, and compliance framework mapping to prioritize remediation.
|
||||
|
||||
This project builds a DLP engine from scratch, teaching you the same detection techniques that power production systems.
|
||||
|
||||
**Real world scenarios where this applies:**
|
||||
- Security engineers scanning file shares before a cloud migration to find PII that needs encryption
|
||||
- Compliance teams auditing database tables for HIPAA-regulated PHI that should not be in plaintext
|
||||
- SOC analysts inspecting PCAP captures for credentials or PII transmitted in the clear
|
||||
- DevOps teams running DLP checks in CI/CD pipelines to catch secrets before they reach production
|
||||
- Incident responders determining what sensitive data was accessible from a compromised network segment
|
||||
|
||||
## What You'll Learn
|
||||
|
||||
**Security Concepts:**
|
||||
- Data classification tiers and how PII, PHI, PCI, and credential data map to regulatory requirements
|
||||
- Confidence scoring: why regex alone produces false positives and how checksum validation, context keywords, and entity co-occurrence reduce them
|
||||
- Compliance framework mapping: HIPAA's 18 identifiers, PCI-DSS cardholder data, GDPR personal data categories, CCPA consumer information
|
||||
- Network DLP: detecting sensitive data in transit, DNS exfiltration via high-entropy subdomain labels, base64-encoded payloads in HTTP bodies
|
||||
- Redaction strategies: why you never store the raw matched content in findings
|
||||
|
||||
**Technical Skills:**
|
||||
- Building a multi-format text extraction pipeline that handles 14+ file formats through a unified Protocol interface
|
||||
- Database schema introspection across 4 database engines with statistical sampling (TABLESAMPLE BERNOULLI, $sample aggregation)
|
||||
- TCP stream reassembly from raw packets using sequence-number ordering and bidirectional flow key normalization
|
||||
- Confidence scoring pipeline: base scores, checksum boosts, context keyword proximity windows, entity co-occurrence
|
||||
- SARIF 2.1.0 output for GitHub code scanning integration
|
||||
|
||||
**Tools and Techniques:**
|
||||
- Typer CLI with Annotated-style parameters and global option propagation through Click context
|
||||
- Pydantic 2.x for configuration validation with YAML loading
|
||||
- structlog with stdlib integration for structured JSON logging
|
||||
- orjson for high-performance JSON serialization
|
||||
- asyncpg, aiomysql, pymongo async, aiosqlite for async database access
|
||||
- dpkt for fast PCAP parsing (100x faster than Scapy)
|
||||
- pytest with hypothesis for property-based testing of detection rules
|
||||
|
||||
## Prerequisites
|
||||
|
||||
**Required knowledge:**
|
||||
- Python fundamentals: dataclasses, type hints, list comprehensions, context managers
|
||||
- Basic networking: TCP/IP, ports, packets, what PCAP files contain
|
||||
- Basic SQL: SELECT, WHERE, table schemas, column types
|
||||
- Security basics: what PII is, why SSNs and credit card numbers need protection, what compliance frameworks exist
|
||||
|
||||
**Tools you'll need:**
|
||||
- Python 3.12+ (uses modern generic syntax and `from __future__ import annotations`)
|
||||
- uv package manager (install: `curl -LsSf https://astral.sh/uv/install.sh | sh`)
|
||||
- A terminal with UTF-8 support (for Rich console output)
|
||||
|
||||
**Helpful but not required:**
|
||||
- Experience with regex and pattern matching
|
||||
- Familiarity with dpkt or Scapy for packet analysis
|
||||
- Knowledge of database URIs and connection strings
|
||||
- Understanding of SARIF format for CI/CD security tooling
|
||||
|
||||
## Quick Start
|
||||
|
||||
```bash
|
||||
bash install.sh
|
||||
dlp-scan file ./data
|
||||
dlp-scan file ./data -f json -o results.json
|
||||
dlp-scan db sqlite:///path/to/database.db
|
||||
dlp-scan report summary results.json
|
||||
```
|
||||
|
|
@ -0,0 +1,133 @@
|
|||
# 01-CONCEPTS.md
|
||||
|
||||
# DLP Concepts
|
||||
|
||||
## What is Data Loss Prevention?
|
||||
|
||||
DLP is the practice of detecting and preventing sensitive data from being stored, transmitted, or accessed in unauthorized ways. The three modes of DLP correspond to the three scan surfaces in this project:
|
||||
|
||||
- **Data at rest**: files on disk, records in databases, documents in cloud storage. Our file scanner and database scanner cover this surface.
|
||||
- **Data in motion**: network traffic, API calls, email transmissions. Our network scanner covers this surface.
|
||||
- **Data in use**: clipboard contents, screen captures, application memory. Not covered here (requires endpoint agents).
|
||||
|
||||
The fundamental question DLP answers: "Where is our sensitive data, and is it protected?"
|
||||
|
||||
## Detection Techniques
|
||||
|
||||
### Pattern Matching with Validation
|
||||
|
||||
The simplest approach: regex patterns that match structural formats like SSNs (XXX-XX-XXXX), credit card numbers (16 digits with known prefixes), and API keys (known prefix patterns like `AKIA` for AWS).
|
||||
|
||||
The problem with regex alone is false positive rates. The string `123-45-6789` matches an SSN pattern but appears in test data, serial numbers, and phone extensions. The string `4532015112830366` matches a Visa card pattern but could be a random 16-digit identifier.
|
||||
|
||||
This is why production DLP systems never rely on regex alone. They add validation layers:
|
||||
|
||||
**Checksum validation** eliminates structurally invalid matches. Credit card numbers use the Luhn algorithm: double every second digit from right, subtract 9 if the result exceeds 9, and verify the total is divisible by 10. A random 16-digit number has a ~10% chance of passing Luhn, which is still useful signal. IBANs use Mod-97 (ISO 7064): rearrange the country code and check digits, convert letters to numbers, and verify the result mod 97 equals 1. NHS numbers use Mod-11 with weighted digit multiplication.
|
||||
|
||||
**SSN area validation** checks that the first three digits are not 000, 666, or 900-999 (never assigned by the SSA). Group and serial numbers must also be non-zero. This eliminates ranges that the Social Security Administration has never used.
|
||||
|
||||
### Context Keyword Scoring
|
||||
|
||||
A 9-digit number matching SSN format near the word "social security" is more likely to be an actual SSN than the same number in a column labeled "serial_number". Context scoring scans a bidirectional window around each match for relevant keywords:
|
||||
|
||||
```
|
||||
For SSN patterns: "ssn", "social security", "social_security_number", "tax id"
|
||||
For credit cards: "credit card", "card number", "payment", "billing"
|
||||
For API keys: "api_key", "secret", "token", "authorization"
|
||||
```
|
||||
|
||||
Keywords found within the window (default: 10 tokens in each direction) add a boost of +0.05 to +0.35 depending on proximity. Closer keywords contribute more confidence.
|
||||
|
||||
### Shannon Entropy
|
||||
|
||||
Random-looking strings often indicate secrets: API keys, encrypted values, base64-encoded credentials. Shannon entropy measures the randomness of a string:
|
||||
|
||||
```
|
||||
H = -sum(p(x) * log2(p(x))) for each unique character x
|
||||
```
|
||||
|
||||
English text has entropy around 3.5-4.5 bits per character. Base64-encoded data is around 5.5-6.0. Hex-encoded data is around 3.5-4.0. Truly random data approaches log2(alphabet_size). A 40-character string with entropy above 4.5 is flagged as a potential secret.
|
||||
|
||||
### Confidence Scoring Pipeline
|
||||
|
||||
Each detection produces a confidence score between 0.0 and 1.0:
|
||||
|
||||
```
|
||||
1. Regex match -> base_score (0.10 to 0.85, configured per rule)
|
||||
2. Checksum validation -> +0.30 if the checksum passes
|
||||
3. Context keyword search -> +0.05 to +0.35 based on keyword proximity
|
||||
4. Entity co-occurrence -> +0.10 to +0.20 if multiple PII types appear nearby
|
||||
5. Final score capped at 1.0
|
||||
```
|
||||
|
||||
The score maps to severity:
|
||||
- 0.85+ = critical
|
||||
- 0.65+ = high
|
||||
- 0.40+ = medium
|
||||
- 0.20+ = low
|
||||
- below 0.20 = discarded
|
||||
|
||||
An SSN match (base 0.45) with valid area/group/serial and the word "ssn" nearby scores 0.45 + 0.30 (area validation acts as implicit checksum) + 0.15 (context) = 0.90, classified as critical. The same pattern without context scores 0.45, classified as medium, which is appropriate because it might be a phone number fragment.
|
||||
|
||||
## Compliance Frameworks
|
||||
|
||||
Regulatory frameworks define what data types require protection and what happens when they are exposed:
|
||||
|
||||
**HIPAA (Health Insurance Portability and Accountability Act)**: Defines 18 types of Protected Health Information (PHI) including SSNs, medical record numbers, health plan beneficiary numbers, and biometric identifiers. A covered entity that fails to protect PHI faces fines from $100 to $50,000 per violation (up to $1.5 million per year per category). The 2015 Anthem breach exposed 78.8 million records and resulted in a $16 million settlement with HHS.
|
||||
|
||||
**PCI-DSS (Payment Card Industry Data Security Standard)**: Requires protection of cardholder data: primary account numbers (PAN), cardholder names, expiration dates, and service codes. PAN must be rendered unreadable (encrypted, hashed, truncated, or tokenized). The Heartland Payment Systems breach (2008) compromised 130 million credit card numbers and cost the company $140 million in compensation.
|
||||
|
||||
**GDPR (General Data Protection Regulation)**: Applies to personal data of EU residents including names, email addresses, phone numbers, IP addresses, and location data. Fines reach 4% of annual global revenue or 20 million euros, whichever is higher. Meta was fined 1.2 billion euros in 2023 for transferring EU user data to the US without adequate safeguards.
|
||||
|
||||
**CCPA (California Consumer Privacy Act)**: Covers personal information of California residents. Similar categories to GDPR but with different enforcement mechanisms. Consumers can sue directly for data breaches involving unencrypted personal information ($100-$750 per consumer per incident).
|
||||
|
||||
## Network DLP Concepts
|
||||
|
||||
### DNS Exfiltration
|
||||
|
||||
Attackers encode stolen data in DNS queries to bypass firewalls that do not inspect DNS traffic. The data is encoded in subdomain labels:
|
||||
|
||||
```
|
||||
aGVsbG8gd29ybGQ.evil.com (base64 "hello world" in subdomain)
|
||||
```
|
||||
|
||||
Detection signals:
|
||||
- **Label entropy**: legitimate subdomains (www, mail, api) have low entropy. Base64-encoded data has entropy above 4.0
|
||||
- **QNAME length**: normal queries are under 50 characters. Exfiltration queries exceed 100+
|
||||
- **TXT query volume**: TXT records are used to receive exfiltrated data. A spike in TXT queries to a single domain is suspicious
|
||||
- **Subdomain label length**: DNS labels above 50 characters are almost never legitimate
|
||||
|
||||
The OilRig APT group (attributed to Iran) used DNS tunneling extensively in campaigns against Middle Eastern governments, encoding stolen documents in subdomain queries to command-and-control infrastructure. DNSCat2 and Iodine are open-source tools that implement this technique.
|
||||
|
||||
### Protocol Identification
|
||||
|
||||
Deep Packet Inspection (DPI) identifies application protocols from payload byte prefixes without relying on port numbers:
|
||||
|
||||
- HTTP requests start with methods: `GET `, `POST `, `PUT `, `DELETE `
|
||||
- HTTP responses start with `HTTP/`
|
||||
- TLS records start with `\x16\x03` (handshake + TLS version)
|
||||
- SSH connections start with `SSH-`
|
||||
- SMTP starts with `220 ` (server greeting)
|
||||
|
||||
This matters because sensitive data in HTTP traffic (API keys in headers, SSNs in POST bodies) requires different handling than the same data in an encrypted TLS stream (where you can only flag that sensitive data was transmitted, not read the content).
|
||||
|
||||
### TCP Stream Reassembly
|
||||
|
||||
Application-layer data spans multiple TCP packets. Reassembly reconstructs the original byte stream:
|
||||
|
||||
1. Track flows by 4-tuple: (src_ip, dst_ip, src_port, dst_port)
|
||||
2. Use bidirectional flow keys so both directions of a conversation map to the same flow
|
||||
3. Store segments indexed by TCP sequence number
|
||||
4. Sort by sequence number and concatenate payloads, deduplicating retransmissions
|
||||
|
||||
Without reassembly, a credit card number split across two packets would be missed by pattern matching on individual payloads.
|
||||
|
||||
## Redaction
|
||||
|
||||
DLP reports must never contain the raw sensitive data they detect. Redaction strategies:
|
||||
|
||||
- **Partial**: preserve structure but mask content: `***-**-6789`, `4532****0366`
|
||||
- **Full**: replace entirely: `[REDACTED]`
|
||||
- **None**: no redaction (for debugging only, never in production reports)
|
||||
|
||||
Partial redaction is preferred for triage because analysts can identify the data type and approximate value without exposing the full sensitive content. The last 4 digits of an SSN or credit card are commonly used as verification tokens and are considered non-sensitive by PCI-DSS.
|
||||
|
|
@ -0,0 +1,539 @@
|
|||
# 02-ARCHITECTURE.md
|
||||
|
||||
# System Architecture
|
||||
|
||||
## High-Level Pipeline
|
||||
|
||||
The scanner follows a linear pipeline: CLI parses arguments, the engine orchestrates, scanners extract and detect, and reporters format output.
|
||||
|
||||
```
|
||||
┌──────────────────────────────────────────────────────────┐
|
||||
│ CLI Layer (Typer) │
|
||||
│ │
|
||||
│ dlp-scan file ./data -f json -o results.json │
|
||||
│ dlp-scan db postgres://user:pass@host/db │
|
||||
│ dlp-scan network capture.pcap │
|
||||
│ dlp-scan report summary results.json │
|
||||
└──────────────────────┬───────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌──────────────────────────────────────────────────────────┐
|
||||
│ ScanEngine │
|
||||
│ │
|
||||
│ Loads config ─► Builds DetectorRegistry ─► Selects │
|
||||
│ scanner type ─► Runs scan ─► Routes to reporter │
|
||||
└──────────────────────┬───────────────────────────────────┘
|
||||
│
|
||||
┌────────────┼────────────┐
|
||||
▼ ▼ ▼
|
||||
┌──────────────┐ ┌──────────┐ ┌──────────────┐
|
||||
│ FileScanner │ │DBScanner │ │NetworkScanner│
|
||||
│ │ │ │ │ │
|
||||
│ Walk dirs │ │ Schema │ │ PCAP parse │
|
||||
│ Extract text │ │ introspect│ │ TCP reassembly│
|
||||
│ Run detectors│ │ Sample │ │ DNS exfil │
|
||||
│ │ │ rows │ │ DPI protocol │
|
||||
│ │ │ Detect │ │ Detect │
|
||||
└──────┬───────┘ └────┬─────┘ └──────┬───────┘
|
||||
│ │ │
|
||||
└──────────────┼──────────────┘
|
||||
▼
|
||||
┌──────────────────────────────────────────────────────────┐
|
||||
│ DetectorRegistry │
|
||||
│ │
|
||||
│ PatternDetector ─► ContextBoost ─► CooccurrenceBoost │
|
||||
│ │ │
|
||||
│ └─► EntropyDetector (parallel) │
|
||||
│ │
|
||||
│ Rules: PII | Financial | Credentials | Health │
|
||||
└──────────────────────┬───────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌──────────────────────────────────────────────────────────┐
|
||||
│ Reporter Layer │
|
||||
│ │
|
||||
│ ConsoleReporter ─► Rich tables with severity colors │
|
||||
│ JsonReporter ─► Structured JSON with metadata │
|
||||
│ SarifReporter ─► SARIF 2.1.0 for CI/CD pipelines │
|
||||
│ CsvReporter ─► Flat CSV for compliance teams │
|
||||
└──────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## Component Breakdown
|
||||
|
||||
### CLI Layer
|
||||
|
||||
**Purpose:** Parse command-line arguments, propagate global options, route to the correct scan command or report utility.
|
||||
|
||||
**Files:** `cli.py`, `commands/scan.py`, `commands/report.py`
|
||||
|
||||
The root Typer app in `cli.py` defines a callback that captures `--config`, `--verbose`, and `--version` into Click's context object. The scan commands (`file`, `db`, `network`) are defined in `commands/scan.py` and registered as top-level commands through a `register(app)` function that calls `app.command("file")(scan_file)` for each. This avoids nesting under a `scan` subgroup while keeping the command definitions in their own module.
|
||||
|
||||
The `report` subgroup is a separate Typer instance added via `app.add_typer(report_app, name="report")`. It provides `convert` (JSON to other formats) and `summary` (print Rich table from JSON results).
|
||||
|
||||
### ScanEngine
|
||||
|
||||
**Purpose:** Single orchestration point that connects config to scanners to reporters.
|
||||
|
||||
**File:** `engine.py`
|
||||
|
||||
The engine takes a `ScanConfig` and constructs a `DetectorRegistry` by unpacking detection configuration into individual parameters:
|
||||
|
||||
```python
|
||||
class ScanEngine:
|
||||
def __init__(self, config: ScanConfig) -> None:
|
||||
self._config = config
|
||||
detection = config.detection
|
||||
allowlist_vals = detection.allowlists.values
|
||||
self._registry = DetectorRegistry(
|
||||
enable_patterns=detection.enable_rules,
|
||||
disable_patterns=detection.disable_rules,
|
||||
allowlist_values=(
|
||||
frozenset(allowlist_vals)
|
||||
if allowlist_vals else None
|
||||
),
|
||||
context_window_tokens=(
|
||||
detection.context_window_tokens
|
||||
),
|
||||
)
|
||||
```
|
||||
|
||||
The engine exposes `scan_files`, `scan_database`, and `scan_network`, each of which constructs the appropriate scanner, runs it, and returns a `ScanResult`. Report generation uses a `REPORTER_MAP` dict that maps format strings to reporter classes.
|
||||
|
||||
### DetectorRegistry
|
||||
|
||||
**Purpose:** Central hub that loads detection rules, filters them by enable/disable globs, and runs the full scoring pipeline against text.
|
||||
|
||||
**File:** `detectors/registry.py`
|
||||
|
||||
The registry loads all rules from four rule modules (PII, Financial, Credentials, Health), filters them using `fnmatch.fnmatch` against enable/disable patterns, and wraps the survivors in a `PatternDetector`. When `detect()` is called:
|
||||
|
||||
1. `PatternDetector` runs all regex patterns, validates matches with checksums (Luhn, Mod-97, Mod-11), and filters against the allowlist
|
||||
2. `apply_context_boost` scans a token window around each match for relevant keywords and adjusts scores based on proximity
|
||||
3. `_apply_cooccurrence_boost` adds a bonus when multiple different PII types appear within 500 characters of each other
|
||||
4. `EntropyDetector` independently finds high-entropy regions using a sliding window
|
||||
|
||||
```
|
||||
Text Input
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────┐
|
||||
│ PatternDetector │
|
||||
│ │
|
||||
│ For each rule: │
|
||||
│ regex.finditer(text) │
|
||||
│ ─► allowlist filter │
|
||||
│ ─► validator (Luhn, │
|
||||
│ Mod-97, SSN area) │
|
||||
│ ─► base_score + boost │
|
||||
└─────────┬───────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────┐
|
||||
│ Context Boost │
|
||||
│ │
|
||||
│ Token window ±10 tokens │
|
||||
│ Keyword proximity search │
|
||||
│ Distance-weighted boost │
|
||||
│ (0.05 to 0.35) │
|
||||
└─────────┬───────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────┐
|
||||
│ Co-occurrence Boost │
|
||||
│ │
|
||||
│ Different rule_ids │
|
||||
│ within 500 chars ─► +0.15│
|
||||
└─────────┬───────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────┐
|
||||
│ Entropy Detector │
|
||||
│ │
|
||||
│ Sliding 256-byte window │
|
||||
│ Shannon H >= 7.2 bits │
|
||||
│ Independent matches │
|
||||
└─────────┬───────────────┘
|
||||
│
|
||||
▼
|
||||
DetectorMatch[]
|
||||
```
|
||||
|
||||
### Scanners
|
||||
|
||||
**Purpose:** Each scanner handles a different scan surface (files, databases, network) and converts raw data into text that the DetectorRegistry can process.
|
||||
|
||||
**Files:** `scanners/file_scanner.py`, `scanners/db_scanner.py`, `scanners/network_scanner.py`
|
||||
|
||||
All scanners follow the same `Scanner` protocol: a `scan(target: str) -> ScanResult` method. They share a common flow: iterate over targets, extract text, run detection, classify matches into findings with severity/compliance/remediation metadata, and aggregate into a `ScanResult`.
|
||||
|
||||
**FileScanner** walks a directory tree, applies extension and exclusion filters, dispatches each file to the appropriate extractor based on extension, and runs the detector on each `TextChunk`. The extension-to-extractor mapping is built once by `_build_extension_map`, which iterates over all extractor instances and indexes by their `supported_extensions`.
|
||||
|
||||
**DatabaseScanner** connects via URI scheme detection (postgres, mysql, mongodb, sqlite), introspects the schema to find text-type columns, samples rows using database-native sampling (TABLESAMPLE BERNOULLI for PostgreSQL, RAND() for MySQL, $sample for MongoDB), and scans column values.
|
||||
|
||||
**NetworkScanner** reads PCAP files using dpkt, extracts TCP/UDP payloads, decodes them to text, and runs detection. The companion modules in `network/` provide TCP stream reassembly, DNS query parsing, protocol identification via DPI, and DNS exfiltration detection.
|
||||
|
||||
### Extractors
|
||||
|
||||
**Purpose:** Convert binary and structured file formats into uniform `TextChunk` objects that carry both the extracted text and a `Location` describing where it came from.
|
||||
|
||||
**Files:** `extractors/plaintext.py`, `extractors/pdf.py`, `extractors/office.py`, `extractors/structured.py`, `extractors/archive.py`, `extractors/email.py`
|
||||
|
||||
All extractors implement the `Extractor` protocol: `extract(path) -> list[TextChunk]` and `supported_extensions -> frozenset[str]`.
|
||||
|
||||
```
|
||||
┌───────────────────────────────────────────────┐
|
||||
│ Extractor Protocol │
|
||||
│ extract(path) -> list[TextChunk] │
|
||||
│ supported_extensions -> frozenset[str] │
|
||||
└───────────────────────────────────────────────┘
|
||||
│
|
||||
┌────┴────┬──────────┬──────────┬──────┐
|
||||
▼ ▼ ▼ ▼ ▼
|
||||
Plaintext PDF Office Structured Archive
|
||||
.txt .log .pdf .docx .csv .json .zip
|
||||
.cfg .py .xlsx .xml .yaml .tar.gz
|
||||
.html .md .xls .parquet .tar.bz2
|
||||
.ts .go .avro
|
||||
... .tsv
|
||||
```
|
||||
|
||||
The `PlaintextExtractor` chunks files into 500-line blocks to keep memory bounded. Binary format extractors (PDF via PyMuPDF, DOCX via python-docx, XLSX via openpyxl) each return one `TextChunk` per page/sheet/section. The archive extractor recurses into compressed files up to a configurable depth with zip bomb protection (compression ratio threshold check).
|
||||
|
||||
### Reporters
|
||||
|
||||
**Purpose:** Take a `ScanResult` and serialize it into the requested output format.
|
||||
|
||||
**Files:** `reporters/console.py`, `reporters/json_report.py`, `reporters/sarif.py`, `reporters/csv_report.py`
|
||||
|
||||
Each reporter has a `generate(result) -> str` method. The `ConsoleReporter` also has a `display(result)` method for Rich-formatted terminal output with severity-colored tables.
|
||||
|
||||
The JSON reporter outputs a structured document with `scan_metadata`, `findings`, and `summary` sections. The SARIF reporter produces a SARIF 2.1.0 document with `tool.driver.rules`, mapping severity levels through `SARIF_SEVERITY_MAP` (critical/high to "error", medium to "warning", low to "note"). The CSV reporter flattens findings into rows.
|
||||
|
||||
## Data Models
|
||||
|
||||
### Core Models
|
||||
|
||||
```python
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Location:
|
||||
source_type: str
|
||||
uri: str
|
||||
line: int | None = None
|
||||
column: int | None = None
|
||||
byte_offset: int | None = None
|
||||
table_name: str | None = None
|
||||
column_name: str | None = None
|
||||
sheet_name: str | None = None
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class Finding:
|
||||
finding_id: str
|
||||
rule_id: str
|
||||
rule_name: str
|
||||
severity: Severity
|
||||
confidence: float
|
||||
location: Location
|
||||
redacted_snippet: str
|
||||
compliance_frameworks: list[str]
|
||||
remediation: str
|
||||
detected_at: datetime
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class ScanResult:
|
||||
scan_id: str
|
||||
tool_version: str
|
||||
scan_started_at: datetime
|
||||
scan_completed_at: datetime | None
|
||||
targets_scanned: int
|
||||
findings: list[Finding]
|
||||
errors: list[str]
|
||||
```
|
||||
|
||||
`Location` is frozen because it represents a fact about where something was found. `Finding` is mutable because fields like `finding_id` and `detected_at` get defaults from factory functions. `ScanResult` aggregates findings and provides computed properties (`findings_by_severity`, `findings_by_rule`, `findings_by_framework`) that group counts for summary reporting.
|
||||
|
||||
The `TextChunk` dataclass carries extracted text paired with its `Location`, forming the bridge between extractors and detectors. Every text fragment knows exactly where it came from, which lets findings carry precise location information through the pipeline.
|
||||
|
||||
### Detection Models
|
||||
|
||||
```python
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class DetectionRule:
|
||||
rule_id: str
|
||||
rule_name: str
|
||||
pattern: re.Pattern[str]
|
||||
base_score: float
|
||||
context_keywords: list[str]
|
||||
validator: Callable[[str], bool] | None
|
||||
compliance_frameworks: list[str]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class DetectorMatch:
|
||||
rule_id: str
|
||||
rule_name: str
|
||||
start: int
|
||||
end: int
|
||||
matched_text: str
|
||||
score: float
|
||||
context_keywords: list[str]
|
||||
compliance_frameworks: list[str]
|
||||
```
|
||||
|
||||
`DetectionRule` is a specification: the regex pattern to match, the base confidence score, optional checksum validator, and context keywords. `DetectorMatch` is a result: what was found, where in the text, and the current score after validation. The `score` field gets modified through the boost pipeline (context, co-occurrence) before being mapped to a `Severity` level and placed into a `Finding`.
|
||||
|
||||
## Configuration Architecture
|
||||
|
||||
```
|
||||
┌────────────────────────────────────────────┐
|
||||
│ .dlp-scanner.yml │
|
||||
│ │
|
||||
│ scan: │
|
||||
│ file: { max_file_size_mb, recursive } │
|
||||
│ database: { sample_percentage } │
|
||||
│ network: { bpf_filter, max_packets } │
|
||||
│ detection: │
|
||||
│ min_confidence, enable_rules, │
|
||||
│ disable_rules, allowlists │
|
||||
│ compliance: { frameworks } │
|
||||
│ output: { format, redaction_style } │
|
||||
│ logging: { level, json_output } │
|
||||
└────────────────┬───────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌────────────────────────────────────────────┐
|
||||
│ load_config(path) -> ScanConfig │
|
||||
│ │
|
||||
│ 1. Check CLI --config flag │
|
||||
│ 2. Search candidates: │
|
||||
│ .dlp-scanner.yml │
|
||||
│ .dlp-scanner.yaml │
|
||||
│ ~/.dlp-scanner.yml │
|
||||
│ 3. Parse YAML via ruamel.yaml │
|
||||
│ 4. Validate with Pydantic 2.x models │
|
||||
│ 5. Return ScanConfig with defaults │
|
||||
└────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
Every configuration value has a constant default defined in `constants.py`. The Pydantic models in `config.py` use these constants as field defaults, so a completely empty config file produces a working scanner. The config loader uses `ruamel.yaml` (not PyYAML) because it preserves comments and handles YAML 1.2.
|
||||
|
||||
The YAML structure uses a `scan:` top-level key to group scanner-specific config, while `detection:`, `compliance:`, `output:`, and `logging:` sit at root level. This mirrors how users think about configuration: "how to scan" vs. "what to detect" vs. "how to report".
|
||||
|
||||
## Data Flow: File Scan
|
||||
|
||||
Step-by-step walkthrough of `dlp-scan file ./data -f json`:
|
||||
|
||||
```
|
||||
1. Typer parses args
|
||||
└─► main() callback stores config_path="" and verbose=False in ctx.obj
|
||||
|
||||
2. scan_file() receives ctx, target="./data", format="json"
|
||||
└─► _run_scan() validates format, loads config, sets logging to WARNING
|
||||
(WARNING for machine-readable formats keeps stdout clean)
|
||||
|
||||
3. ScanEngine(config) constructs DetectorRegistry
|
||||
└─► Registry loads 28 rules from PII/Financial/Credential/Health modules
|
||||
└─► Filters through enable_rules=["*"], disable_rules=[]
|
||||
|
||||
4. engine.scan_files("./data")
|
||||
└─► FileScanner.scan() creates ScanResult, walks directory
|
||||
|
||||
5. For each file in ./data/**/*:
|
||||
└─► Check extension against include_extensions
|
||||
└─► Check path against exclude_patterns
|
||||
└─► Check file size against max_file_size_mb
|
||||
└─► Select extractor by extension (e.g. .csv -> CsvExtractor)
|
||||
└─► extractor.extract(path) -> list[TextChunk]
|
||||
|
||||
6. For each TextChunk:
|
||||
└─► registry.detect(chunk.text) -> list[DetectorMatch]
|
||||
├─► PatternDetector: regex match + allowlist + validator
|
||||
├─► apply_context_boost: keyword proximity scoring
|
||||
├─► _apply_cooccurrence_boost: multi-PII bonus
|
||||
└─► EntropyDetector: high-entropy region detection
|
||||
|
||||
7. For each DetectorMatch above min_confidence:
|
||||
└─► score_to_severity(match.score) -> Severity
|
||||
└─► get_frameworks_for_rule(match.rule_id) -> compliance list
|
||||
└─► get_remediation_for_rule(match.rule_id) -> guidance string
|
||||
└─► redact(chunk.text, start, end, style="partial") -> snippet
|
||||
└─► Append Finding to ScanResult
|
||||
|
||||
8. Back in _run_scan():
|
||||
└─► engine.generate_report(result, "json")
|
||||
└─► JsonReporter().generate(result) -> JSON string
|
||||
└─► typer.echo(output) -> stdout
|
||||
```
|
||||
|
||||
## Design Patterns
|
||||
|
||||
### Protocol-Based Polymorphism
|
||||
|
||||
The codebase uses Python's `typing.Protocol` instead of abstract base classes for extension points. The `Extractor`, `Scanner`, and `Detector` protocols define structural interfaces without requiring inheritance.
|
||||
|
||||
```python
|
||||
class Extractor(Protocol):
|
||||
def extract(self, path: str) -> list[TextChunk]: ...
|
||||
|
||||
@property
|
||||
def supported_extensions(self) -> frozenset[str]: ...
|
||||
```
|
||||
|
||||
Any class with matching method signatures satisfies the protocol. This means you can add a new extractor (say, for .pptx files) without importing the base module. The type checker verifies compliance; the runtime never checks inheritance.
|
||||
|
||||
**Why not ABCs:** Abstract base classes force an import dependency and mandate `super().__init__()` chains. Protocols are lighter and match Python's duck typing philosophy. Since extractors are stateless (no shared state or lifecycle), there is nothing an ABC would provide beyond the type contract.
|
||||
|
||||
### Registry Pattern
|
||||
|
||||
The `DetectorRegistry` centralizes rule management: loading, filtering, and execution. Individual rule modules (pii.py, financial.py, credentials.py, health.py) each export a list of `DetectionRule` objects. The registry merges them into `ALL_RULES`, applies glob filtering, and wraps the result in a `PatternDetector`.
|
||||
|
||||
This keeps rule definitions declarative. Adding a new rule is a matter of appending a `DetectionRule` to the appropriate list. The registry handles filtering and execution without rule authors needing to understand the scoring pipeline.
|
||||
|
||||
### Command Registration Pattern
|
||||
|
||||
CLI commands are defined in `commands/scan.py` as plain functions and registered on the root app through a `register(app)` function:
|
||||
|
||||
```python
|
||||
def register(app: typer.Typer) -> None:
|
||||
app.command("file")(scan_file)
|
||||
app.command("db")(scan_db)
|
||||
app.command("network")(scan_network)
|
||||
```
|
||||
|
||||
This achieves top-level commands (`dlp-scan file`, not `dlp-scan scan file`) while keeping the command logic out of `cli.py`. The `_run_scan` helper deduplicates the shared logic (config loading, format validation, output routing) across all three scan types.
|
||||
|
||||
## Compliance Mapping
|
||||
|
||||
The compliance module maps rule IDs to regulatory frameworks and remediation guidance using two static dictionaries:
|
||||
|
||||
```
|
||||
RULE_FRAMEWORK_MAP: rule_id -> [frameworks]
|
||||
RULE_REMEDIATION_MAP: rule_id -> guidance string
|
||||
```
|
||||
|
||||
When a `DetectorMatch` is converted to a `Finding` inside a scanner, the scanner calls `get_frameworks_for_rule` and `get_remediation_for_rule` to decorate the finding with compliance metadata. If the detection rule itself also carries `compliance_frameworks`, both sets are merged.
|
||||
|
||||
This design keeps detection rules independent of compliance logic. The PII module does not need to know that HIPAA cares about SSNs. The compliance module owns that mapping, and it can be updated independently when regulations change.
|
||||
|
||||
## Redaction Pipeline
|
||||
|
||||
```
|
||||
matched text
|
||||
│
|
||||
▼
|
||||
style == "none"? ─yes─► raw snippet with context
|
||||
│ no
|
||||
▼
|
||||
style == "full"? ─yes─► [REDACTED] with context
|
||||
│ no
|
||||
▼
|
||||
_partial_redact()
|
||||
│
|
||||
├─ 9+ digit number ─► *****6789 (mask all but last 4)
|
||||
├─ email address ─► j****@example.com
|
||||
└─ generic string ─► keep last 25%
|
||||
│
|
||||
▼
|
||||
_build_snippet()
|
||||
│
|
||||
└─ ±20 chars context ─► "...SSN: *****6789 for..."
|
||||
```
|
||||
|
||||
Partial redaction is the default because it gives analysts enough to identify the data type and triage priority without exposing the full sensitive value. The last 4 digits of SSNs and credit cards are considered non-sensitive by PCI-DSS (you can print them on receipts), so partial redaction for those types is compliant.
|
||||
|
||||
## Network Analysis Architecture
|
||||
|
||||
```
|
||||
┌────────────────────────────────────────────┐
|
||||
│ PCAP File │
|
||||
│ (.pcap or .pcapng) │
|
||||
└────────────────┬───────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌────────────────────────────────────────────┐
|
||||
│ pcap.read_pcap() │
|
||||
│ │
|
||||
│ dpkt.pcap.Reader / dpkt.pcapng.Reader │
|
||||
│ Parse Ethernet -> IP -> TCP/UDP │
|
||||
│ Yield PacketInfo(src_ip, dst_ip, │
|
||||
│ src_port, dst_port, payload, │
|
||||
│ tcp_seq, tcp_flags) │
|
||||
└────────────────┬───────────────────────────┘
|
||||
│
|
||||
┌───────┴───────┐
|
||||
▼ ▼
|
||||
┌─────────────┐ ┌───────────────┐
|
||||
│FlowTracker │ │DnsExfilDetector│
|
||||
│ │ │ │
|
||||
│Track by │ │Label length │
|
||||
│4-tuple key │ │check (>50) │
|
||||
│ │ │ │
|
||||
│Reassemble │ │Subdomain │
|
||||
│TCP streams │ │entropy (>4.0) │
|
||||
│by seq num │ │ │
|
||||
│ │ │QNAME length │
|
||||
│Dedup retx │ │check (>100) │
|
||||
└──────┬──────┘ │ │
|
||||
│ │TXT volume │
|
||||
▼ │ratio check │
|
||||
┌─────────────┐ └───────┬───────┘
|
||||
│Protocol ID │ │
|
||||
│(DPI) │ ▼
|
||||
│ │ ExfilIndicator[]
|
||||
│HTTP: method │
|
||||
│ prefix │
|
||||
│TLS: \x16\x03│
|
||||
│SSH: SSH- │
|
||||
│SMTP: 220 │
|
||||
└──────┬──────┘
|
||||
│
|
||||
▼
|
||||
Reassembled text
|
||||
sent to DetectorRegistry
|
||||
```
|
||||
|
||||
The flow tracker creates bidirectional flow keys by sorting the forward and reverse 4-tuples, so `(A, B, 80, 12345)` and `(B, A, 12345, 80)` map to the same flow. TCP reassembly sorts segments by sequence number and deduplicates retransmissions. Without reassembly, a credit card number split across two TCP segments would be missed.
|
||||
|
||||
The DNS exfiltration detector runs independently of the regex-based detectors. It analyzes DNS queries for encoding signals: base64-like entropy in subdomain labels, abnormally long labels, long QNAMEs, and suspicious TXT query volume ratios. The OilRig APT campaign used exactly these patterns to exfiltrate stolen documents through DNS tunneling to C2 infrastructure.
|
||||
|
||||
## Error Handling Strategy
|
||||
|
||||
Errors are collected, not thrown. Each scanner appends error messages to `ScanResult.errors` and continues scanning the remaining targets. The CLI checks `result.errors` after the scan completes and exits with code 1 if any errors occurred, but the partial results are still reported.
|
||||
|
||||
This "collect and continue" approach means a single corrupt PDF in a directory of 10,000 files does not abort the scan. The Equifax breach investigation found that scanning tools that failed on individual files often left entire directories unscanned, which is why modern DLP tools treat extraction failures as warnings rather than fatal errors.
|
||||
|
||||
## Performance Considerations
|
||||
|
||||
**File scanning** is I/O-bound. The scanner processes files sequentially to avoid overwhelming disk I/O. Text extraction for binary formats (PDF, Office) can be CPU-intensive, but these files are typically a small fraction of the total.
|
||||
|
||||
**Detection** scales linearly with text length times rule count. With 28 rules and an average text chunk of 500 lines, a single detection pass takes microseconds. The entropy detector is more expensive due to its sliding window, so it only runs when enabled and only against high-level text chunks (not individual regex matches).
|
||||
|
||||
**Memory** stays bounded through chunking. The plaintext extractor reads 500 lines at a time. Archive extraction enforces depth limits and zip bomb ratio checks.
|
||||
|
||||
## Key Files Reference
|
||||
|
||||
- `cli.py` - Entry point, global options, Typer app
|
||||
- `engine.py` - Orchestration, connects config to scanners to reporters
|
||||
- `config.py` - Pydantic models, YAML loading, config search
|
||||
- `constants.py` - All magic numbers, thresholds, type literals
|
||||
- `models.py` - Finding, Location, ScanResult, TextChunk
|
||||
- `compliance.py` - Rule-to-framework mapping, severity classification
|
||||
- `redaction.py` - Partial/full/none redaction strategies
|
||||
- `detectors/registry.py` - Rule loading, filtering, scoring pipeline
|
||||
- `detectors/pattern.py` - Regex matching with allowlist and checksum validation
|
||||
- `detectors/context.py` - Keyword proximity boost, co-occurrence boost
|
||||
- `detectors/entropy.py` - Shannon entropy detection, sliding window
|
||||
- `detectors/rules/` - Rule definitions (pii, financial, credentials, health)
|
||||
- `extractors/` - Text extraction from 14+ file formats
|
||||
- `scanners/` - File, database, network scan implementations
|
||||
- `network/` - PCAP parsing, flow tracking, DPI, DNS exfiltration
|
||||
- `reporters/` - Console, JSON, SARIF, CSV output
|
||||
- `commands/` - CLI command implementations (scan, report)
|
||||
|
||||
## Next Steps
|
||||
|
||||
Now that you understand the architecture:
|
||||
1. Read [03-IMPLEMENTATION.md](./03-IMPLEMENTATION.md) for the code walkthrough
|
||||
2. Try modifying a detection rule in `detectors/rules/pii.py` to see how the scoring pipeline responds
|
||||
|
|
@ -0,0 +1,801 @@
|
|||
# 03-IMPLEMENTATION.md
|
||||
|
||||
# Implementation Guide
|
||||
|
||||
This document walks through how the code works. We cover the detection engine, file extraction, network analysis, and CLI integration, with code snippets from the actual project.
|
||||
|
||||
## File Structure
|
||||
|
||||
```
|
||||
src/dlp_scanner/
|
||||
├── __init__.py
|
||||
├── cli.py # Typer entry point
|
||||
├── engine.py # Scan orchestration
|
||||
├── config.py # Pydantic config models
|
||||
├── constants.py # Thresholds, types, defaults
|
||||
├── models.py # Finding, Location, ScanResult
|
||||
├── compliance.py # Rule-to-framework mapping
|
||||
├── redaction.py # Snippet masking
|
||||
├── log.py # structlog configuration
|
||||
├── commands/
|
||||
│ ├── scan.py # file, db, network commands
|
||||
│ └── report.py # convert, summary commands
|
||||
├── detectors/
|
||||
│ ├── base.py # DetectionRule, DetectorMatch
|
||||
│ ├── pattern.py # Regex + checksum detection
|
||||
│ ├── context.py # Keyword proximity scoring
|
||||
│ ├── entropy.py # Shannon entropy detection
|
||||
│ ├── registry.py # Central detector registry
|
||||
│ └── rules/
|
||||
│ ├── pii.py # SSN, email, phone, passport
|
||||
│ ├── financial.py # Credit cards, IBAN, NHS
|
||||
│ ├── credentials.py # AWS, GitHub, JWT, Stripe
|
||||
│ └── health.py # Medical records, DEA, NPI
|
||||
├── extractors/
|
||||
│ ├── base.py # Extractor protocol
|
||||
│ ├── plaintext.py # .txt, .log, .cfg, source code
|
||||
│ ├── pdf.py # .pdf via PyMuPDF
|
||||
│ ├── office.py # .docx, .xlsx, .xls
|
||||
│ ├── structured.py # .csv, .json, .xml, .yaml, .parquet, .avro
|
||||
│ ├── archive.py # .zip, .tar.gz, .tar.bz2
|
||||
│ └── email.py # .eml, .msg
|
||||
├── network/
|
||||
│ ├── pcap.py # PCAP/PCAPNG packet reader
|
||||
│ ├── flow_tracker.py # TCP flow reassembly
|
||||
│ ├── protocols.py # DPI protocol identification
|
||||
│ └── exfiltration.py # DNS exfil detection
|
||||
├── reporters/
|
||||
│ ├── base.py # Reporter protocol
|
||||
│ ├── console.py # Rich terminal output
|
||||
│ ├── json_report.py # Structured JSON
|
||||
│ ├── sarif.py # SARIF 2.1.0
|
||||
│ └── csv_report.py # Flat CSV
|
||||
└── scanners/
|
||||
├── base.py # Scanner protocol
|
||||
├── file_scanner.py # Directory walking + extraction
|
||||
├── db_scanner.py # DB schema introspection
|
||||
└── network_scanner.py # PCAP payload scanning
|
||||
```
|
||||
|
||||
## Building the Detection Engine
|
||||
|
||||
### Detection Rules
|
||||
|
||||
Every detection rule is a data structure, not a class hierarchy. The `DetectionRule` dataclass holds the regex pattern, base confidence score, optional validator function, context keywords, and compliance framework tags:
|
||||
|
||||
```python
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class DetectionRule:
|
||||
rule_id: str
|
||||
rule_name: str
|
||||
pattern: re.Pattern[str]
|
||||
base_score: float
|
||||
context_keywords: list[str] = field(default_factory=list)
|
||||
validator: Callable[[str], bool] | None = None
|
||||
compliance_frameworks: list[str] = field(
|
||||
default_factory=list
|
||||
)
|
||||
```
|
||||
|
||||
Rule modules export plain lists of these structs. Here is the SSN rule from `detectors/rules/pii.py`:
|
||||
|
||||
```python
|
||||
SSN_PATTERN = re.compile(
|
||||
r"\b(?!000|666|9\d{2})\d{3}"
|
||||
r"[-\s]?"
|
||||
r"(?!00)\d{2}"
|
||||
r"[-\s]?"
|
||||
r"(?!0000)\d{4}\b"
|
||||
)
|
||||
|
||||
PII_RULES: list[DetectionRule] = [
|
||||
DetectionRule(
|
||||
rule_id="PII_SSN",
|
||||
rule_name="US Social Security Number",
|
||||
pattern=SSN_PATTERN,
|
||||
base_score=0.45,
|
||||
context_keywords=SSN_CONTEXT,
|
||||
validator=_validate_ssn,
|
||||
compliance_frameworks=[
|
||||
"HIPAA", "CCPA", "GLBA", "GDPR",
|
||||
],
|
||||
),
|
||||
...
|
||||
]
|
||||
```
|
||||
|
||||
The regex uses negative lookaheads (`(?!000|666|9\d{2})`) to reject SSN area numbers the Social Security Administration has never assigned. This is a first-pass structural filter. The real validation happens in `_validate_ssn`, which the `PatternDetector` calls for every regex match.
|
||||
|
||||
**Why base_score is 0.45, not higher:** A 9-digit number matching the SSN format appears in serial numbers, zip+4 codes, phone fragments, and test data constantly. The string `456-78-9012` matches the SSN pattern and passes area/group/serial validation, but without context it could be anything. A base of 0.45 keeps it in the "medium" severity tier until context boosts push it higher.
|
||||
|
||||
### Checksum Validation
|
||||
|
||||
The three checksum validators demonstrate different mathematical approaches to the same problem: distinguishing real identifiers from random digit sequences.
|
||||
|
||||
**Luhn algorithm** for credit cards (in `detectors/rules/financial.py`):
|
||||
|
||||
```python
|
||||
def luhn_check(number: str) -> bool:
|
||||
digits = [int(d) for d in number if d.isdigit()]
|
||||
if len(digits) < 13:
|
||||
return False
|
||||
|
||||
odd_digits = digits[-1::-2]
|
||||
even_digits = digits[-2::-2]
|
||||
total = sum(odd_digits)
|
||||
for d in even_digits:
|
||||
total += sum(divmod(d * 2, 10))
|
||||
return total % 10 == 0
|
||||
```
|
||||
|
||||
The algorithm works right-to-left: take every other digit starting from the rightmost, sum them. For the remaining digits, double each, and if the result exceeds 9, subtract 9 (which is what `sum(divmod(d * 2, 10))` does). If the grand total is divisible by 10, the number is valid. A random 16-digit number has about a 10% chance of passing Luhn, so it reduces false positives by roughly 90%.
|
||||
|
||||
**Mod-97** for IBANs (ISO 7064):
|
||||
|
||||
```python
|
||||
def iban_check(value: str) -> bool:
|
||||
cleaned = value.replace(" ", "").upper()
|
||||
if len(cleaned) < 15 or len(cleaned) > 34:
|
||||
return False
|
||||
|
||||
rearranged = cleaned[4:] + cleaned[:4]
|
||||
numeric = ""
|
||||
for char in rearranged:
|
||||
if char.isalpha():
|
||||
numeric += str(ord(char) - ord("A") + 10)
|
||||
else:
|
||||
numeric += char
|
||||
|
||||
return int(numeric) % 97 == 1
|
||||
```
|
||||
|
||||
Move the country code and check digits (first 4 chars) to the end, convert letters to two-digit numbers (A=10, B=11, etc.), then check that the entire number mod 97 equals 1. The false positive rate is approximately 1 in 97.
|
||||
|
||||
**Mod-11** for NHS numbers:
|
||||
|
||||
```python
|
||||
def nhs_check(value: str) -> bool:
|
||||
digits = value.replace("-", "").replace(" ", "")
|
||||
if len(digits) != 10 or not digits.isdigit():
|
||||
return False
|
||||
|
||||
weights = range(10, 1, -1)
|
||||
total = sum(
|
||||
int(d) * w
|
||||
for d, w in zip(digits[:9], weights, strict=False)
|
||||
)
|
||||
remainder = 11 - (total % 11)
|
||||
if remainder == 11:
|
||||
remainder = 0
|
||||
if remainder == 10:
|
||||
return False
|
||||
return remainder == int(digits[9])
|
||||
```
|
||||
|
||||
Multiply the first 9 digits by descending weights (10, 9, 8, ..., 2), sum them, compute `11 - (sum mod 11)`, and compare to the check digit. If the result is 10, the number is invalid (NHS never issues these). If the result is 11, the check digit is 0.
|
||||
|
||||
### Pattern Detection
|
||||
|
||||
The `PatternDetector` in `detectors/pattern.py` iterates over all active rules, runs each regex against the input text, filters through the allowlist, and applies checksum validation:
|
||||
|
||||
```python
|
||||
class PatternDetector:
|
||||
def detect(self, text: str) -> list[DetectorMatch]:
|
||||
matches: list[DetectorMatch] = []
|
||||
|
||||
for rule in self._rules:
|
||||
for m in rule.pattern.finditer(text):
|
||||
matched_text = m.group()
|
||||
|
||||
if self._is_allowlisted(matched_text):
|
||||
continue
|
||||
|
||||
score = rule.base_score
|
||||
|
||||
if rule.validator is not None:
|
||||
if rule.validator(matched_text):
|
||||
score = min(1.0, score + CHECKSUM_BOOST)
|
||||
else:
|
||||
continue
|
||||
|
||||
matches.append(
|
||||
DetectorMatch(
|
||||
rule_id=rule.rule_id,
|
||||
...
|
||||
score=score,
|
||||
)
|
||||
)
|
||||
|
||||
return matches
|
||||
```
|
||||
|
||||
When a rule has a validator and the match fails validation, the match is discarded entirely (`continue`). A Visa pattern that matches `4532015112830366` but fails Luhn is not a credit card. When validation passes, the score gets a +0.30 boost (`CHECKSUM_BOOST`). This is aggressive because checksum-passing matches are overwhelmingly real: the Luhn+Visa prefix combination has a false positive rate under 1%.
|
||||
|
||||
The allowlist uses a frozen set lookup, defaulting to `KNOWN_TEST_VALUES` (common test card numbers, example SSNs like `123-45-6789`). This prevents DLP tools from flagging their own test data, which is a common complaint in production deployments.
|
||||
|
||||
### Context Keyword Scoring
|
||||
|
||||
After pattern detection, `apply_context_boost` in `detectors/context.py` scans the surrounding text for keywords that indicate the matched value is actually sensitive data:
|
||||
|
||||
```python
|
||||
def apply_context_boost(
|
||||
text: str,
|
||||
matches: list[DetectorMatch],
|
||||
window_tokens: int = DEFAULT_CONTEXT_WINDOW_TOKENS,
|
||||
) -> list[DetectorMatch]:
|
||||
tokens = text.lower().split()
|
||||
boosted: list[DetectorMatch] = []
|
||||
|
||||
for match in matches:
|
||||
if not match.context_keywords:
|
||||
boosted.append(match)
|
||||
continue
|
||||
|
||||
char_to_token = _char_offset_to_token_index(
|
||||
text, match.start
|
||||
)
|
||||
window_start = max(
|
||||
0, char_to_token - window_tokens
|
||||
)
|
||||
window_end = min(
|
||||
len(tokens), char_to_token + window_tokens
|
||||
)
|
||||
window_text = " ".join(
|
||||
tokens[window_start:window_end]
|
||||
)
|
||||
|
||||
boost = _compute_keyword_boost(
|
||||
window_text,
|
||||
match.context_keywords,
|
||||
window_tokens,
|
||||
)
|
||||
|
||||
new_score = min(1.0, match.score + boost)
|
||||
...
|
||||
```
|
||||
|
||||
The window is bidirectional: 10 tokens in each direction from the match. The boost is distance-weighted: a keyword right next to the match contributes up to `CONTEXT_BOOST_MAX` (0.35), while one at the edge of the window contributes almost nothing. This reflects a real observation: "SSN: 456-78-9012" is almost certainly an SSN, while "SSN" appearing 50 words away from "456-78-9012" is weaker signal.
|
||||
|
||||
The `_compute_keyword_boost` function finds the best keyword match in the window and computes `CONTEXT_BOOST_MAX * proximity_factor`, where proximity is `1.0 - (distance / max_distance)`. Only the highest-scoring keyword matters, not the sum of all keywords. This prevents keyword stuffing from inflating scores.
|
||||
|
||||
### Co-occurrence Boost
|
||||
|
||||
After context boosting, `_apply_cooccurrence_boost` checks whether multiple different PII types appear near each other:
|
||||
|
||||
```python
|
||||
def _apply_cooccurrence_boost(
|
||||
matches: list[DetectorMatch],
|
||||
) -> list[DetectorMatch]:
|
||||
if len(matches) < 2:
|
||||
return matches
|
||||
|
||||
proximity_threshold = 500
|
||||
|
||||
for i, match in enumerate(matches):
|
||||
has_neighbor = False
|
||||
for j, other in enumerate(matches):
|
||||
if i == j:
|
||||
continue
|
||||
if other.rule_id == match.rule_id:
|
||||
continue
|
||||
distance = abs(match.start - other.start)
|
||||
if distance < proximity_threshold:
|
||||
has_neighbor = True
|
||||
break
|
||||
|
||||
if has_neighbor:
|
||||
new_score = min(
|
||||
1.0, match.score + COOCCURRENCE_BOOST
|
||||
)
|
||||
...
|
||||
```
|
||||
|
||||
An SSN near a credit card number is stronger evidence than either alone. The boost is +0.15 (`COOCCURRENCE_BOOST`), and it requires different `rule_id` values (two SSNs next to each other do not trigger it). The 500-character threshold roughly corresponds to a short paragraph or a few database columns.
|
||||
|
||||
This heuristic matters in practice. The Capital One breach data contained CSV exports where SSNs, credit card numbers, and addresses appeared in adjacent columns. Co-occurrence detection would have flagged these files as critical priority.
|
||||
|
||||
### Shannon Entropy Detection
|
||||
|
||||
The `EntropyDetector` in `detectors/entropy.py` finds high-entropy regions that may contain secrets, encrypted data, or base64-encoded credentials:
|
||||
|
||||
```python
|
||||
def shannon_entropy(data: bytes) -> float:
|
||||
if not data:
|
||||
return 0.0
|
||||
|
||||
counts = Counter(data)
|
||||
total = len(data)
|
||||
return -sum(
|
||||
(c / total) * math.log2(c / total)
|
||||
for c in counts.values()
|
||||
)
|
||||
```
|
||||
|
||||
Shannon entropy measures the average information content per byte. English text sits around 3.5-4.5 bits. Base64-encoded data is 5.5-6.0. Truly random bytes approach 8.0 (log2(256)). The detector uses a sliding window of 256 bytes with a 128-byte step:
|
||||
|
||||
```python
|
||||
def detect_high_entropy_regions(
|
||||
data: bytes,
|
||||
threshold: float = DEFAULT_ENTROPY_THRESHOLD,
|
||||
window_size: int = WINDOW_SIZE,
|
||||
step: int = WINDOW_STEP,
|
||||
) -> list[tuple[int, int, float]]:
|
||||
...
|
||||
while i + window_size <= len(data):
|
||||
window = data[i:i + window_size]
|
||||
h = shannon_entropy(window)
|
||||
|
||||
if h >= threshold:
|
||||
end = i + window_size
|
||||
while end + step <= len(data):
|
||||
next_window = data[
|
||||
end - window_size + step:end + step
|
||||
]
|
||||
next_h = shannon_entropy(next_window)
|
||||
if next_h < threshold:
|
||||
break
|
||||
h = max(h, next_h)
|
||||
end += step
|
||||
|
||||
regions.append((i, end, h))
|
||||
i = end
|
||||
else:
|
||||
i += step
|
||||
```
|
||||
|
||||
When the entropy exceeds the threshold (default 7.2), the detector extends the region forward until entropy drops below the threshold. This merges adjacent high-entropy windows into a single region rather than reporting dozens of overlapping detections.
|
||||
|
||||
The default threshold of 7.2 is intentionally high. Network payloads containing binary protocol data or compressed content often hit 6.0-7.0, which would generate massive false positive volume. At 7.2, the detector primarily catches encrypted blobs, base64-encoded secrets, and random key material.
|
||||
|
||||
## File Extraction Pipeline
|
||||
|
||||
### The Extractor Protocol
|
||||
|
||||
All extractors implement a two-method protocol:
|
||||
|
||||
```python
|
||||
class Extractor(Protocol):
|
||||
def extract(self, path: str) -> list[TextChunk]: ...
|
||||
|
||||
@property
|
||||
def supported_extensions(self) -> frozenset[str]: ...
|
||||
```
|
||||
|
||||
The `FileScanner` builds an extension-to-extractor map at initialization by iterating over all extractor instances and indexing by their supported extensions. When scanning a file, it looks up the extractor by the file's extension and calls `extract`.
|
||||
|
||||
### Plaintext Extraction
|
||||
|
||||
The `PlaintextExtractor` reads files in 500-line chunks to keep memory bounded:
|
||||
|
||||
```python
|
||||
class PlaintextExtractor:
|
||||
def extract(self, path: str) -> list[TextChunk]:
|
||||
chunks: list[TextChunk] = []
|
||||
|
||||
with open(
|
||||
path, encoding="utf-8", errors="replace",
|
||||
) as f:
|
||||
lines: list[str] = []
|
||||
line_number = 1
|
||||
chunk_start = 1
|
||||
|
||||
for line in f:
|
||||
lines.append(line)
|
||||
if len(lines) >= CHUNK_MAX_LINES:
|
||||
chunks.append(
|
||||
TextChunk(
|
||||
text="".join(lines),
|
||||
location=Location(
|
||||
source_type="file",
|
||||
uri=path,
|
||||
line=chunk_start,
|
||||
),
|
||||
)
|
||||
)
|
||||
chunk_start = line_number + 1
|
||||
lines = []
|
||||
line_number += 1
|
||||
|
||||
if lines:
|
||||
chunks.append(...)
|
||||
|
||||
return chunks
|
||||
```
|
||||
|
||||
Each `TextChunk` carries the starting line number in its `Location`, so findings can report where in the file the match occurred. The `errors="replace"` parameter means binary-contaminated text files (common in log files with embedded binary data) will not crash the extractor.
|
||||
|
||||
### Extension Map Construction
|
||||
|
||||
The `_build_extension_map` function in `file_scanner.py` constructs the mapping from extensions to extractors:
|
||||
|
||||
```python
|
||||
def _build_extension_map() -> dict[str, Extractor]:
|
||||
extractors: list[Extractor] = [
|
||||
PlaintextExtractor(),
|
||||
PDFExtractor(),
|
||||
DocxExtractor(),
|
||||
XlsxExtractor(),
|
||||
XlsExtractor(),
|
||||
CsvExtractor(),
|
||||
JsonExtractor(),
|
||||
XmlExtractor(),
|
||||
YamlExtractor(),
|
||||
ParquetExtractor(),
|
||||
AvroExtractor(),
|
||||
ArchiveExtractor(),
|
||||
EmlExtractor(),
|
||||
MsgExtractor(),
|
||||
]
|
||||
|
||||
ext_map: dict[str, Extractor] = {}
|
||||
for extractor in extractors:
|
||||
for ext in extractor.supported_extensions:
|
||||
ext_map[ext] = extractor
|
||||
|
||||
return ext_map
|
||||
```
|
||||
|
||||
Adding a new format means creating an extractor class with `extract` and `supported_extensions`, then adding it to this list. The scanner does not need to know anything about the format.
|
||||
|
||||
### File Scanner Walk Logic
|
||||
|
||||
The `FileScanner._scan_directory` method applies a chain of filters before dispatching to an extractor:
|
||||
|
||||
```python
|
||||
def _scan_directory(self, directory, result):
|
||||
iterator = (
|
||||
directory.rglob("*")
|
||||
if self._file_config.recursive
|
||||
else directory.glob("*")
|
||||
)
|
||||
|
||||
for path in iterator:
|
||||
if not path.is_file():
|
||||
continue
|
||||
if self._is_excluded(path, directory):
|
||||
continue
|
||||
|
||||
suffix = _get_full_suffix(path)
|
||||
if suffix not in self._allowed_extensions:
|
||||
continue
|
||||
|
||||
file_size = path.stat().st_size
|
||||
if file_size > max_bytes:
|
||||
continue
|
||||
if file_size == 0:
|
||||
continue
|
||||
|
||||
self._scan_file(path, result)
|
||||
result.targets_scanned += 1
|
||||
```
|
||||
|
||||
The `_get_full_suffix` function handles compound extensions like `.tar.gz` and `.tar.bz2` by checking the filename suffix before falling back to `path.suffix.lower()`. The exclusion check matches against the relative path, the filename, and individual path components, so a pattern like `__pycache__` matches regardless of depth.
|
||||
|
||||
## Network Analysis
|
||||
|
||||
### PCAP Parsing
|
||||
|
||||
The `read_pcap` function in `network/pcap.py` reads packets using dpkt and yields `PacketInfo` structs:
|
||||
|
||||
```python
|
||||
def read_pcap(path, max_packets=0):
|
||||
with open(path, "rb") as f:
|
||||
try:
|
||||
pcap = dpkt.pcap.Reader(f)
|
||||
except ValueError:
|
||||
f.seek(0)
|
||||
pcap = dpkt.pcapng.Reader(f)
|
||||
|
||||
count = 0
|
||||
for timestamp, buf in pcap:
|
||||
if max_packets > 0 and count >= max_packets:
|
||||
break
|
||||
|
||||
packet = _parse_ethernet(timestamp, buf)
|
||||
if packet is not None:
|
||||
yield packet
|
||||
count += 1
|
||||
```
|
||||
|
||||
The try/except fallback handles both PCAP (libpcap) and PCAPNG (Wireshark's newer format). dpkt is used instead of Scapy because it is roughly 100x faster for bulk packet parsing. Scapy constructs rich protocol objects with dissection layers; dpkt does minimal parsing and gives you raw bytes.
|
||||
|
||||
### TCP Flow Reassembly
|
||||
|
||||
The `FlowTracker` in `network/flow_tracker.py` groups packets into flows and reassembles TCP streams:
|
||||
|
||||
```python
|
||||
def make_flow_key(packet):
|
||||
forward = (
|
||||
packet.src_ip, packet.dst_ip,
|
||||
packet.src_port, packet.dst_port,
|
||||
)
|
||||
reverse = (
|
||||
packet.dst_ip, packet.src_ip,
|
||||
packet.dst_port, packet.src_port,
|
||||
)
|
||||
return min(forward, reverse)
|
||||
```
|
||||
|
||||
The bidirectional key is the lexicographically smaller of the forward and reverse 4-tuples. This means `(A->B)` and `(B->A)` packets land in the same flow. The `reassemble_stream` method sorts segments by TCP sequence number and deduplicates retransmissions:
|
||||
|
||||
```python
|
||||
def reassemble_stream(self, key):
|
||||
flow = self._flows.get(key)
|
||||
if flow is None:
|
||||
return b""
|
||||
|
||||
sorted_segments = sorted(
|
||||
flow.segments, key=lambda s: s[0]
|
||||
)
|
||||
|
||||
seen_offsets: set[int] = set()
|
||||
parts: list[bytes] = []
|
||||
for seq, data in sorted_segments:
|
||||
if seq not in seen_offsets:
|
||||
seen_offsets.add(seq)
|
||||
parts.append(data)
|
||||
|
||||
return b"".join(parts)
|
||||
```
|
||||
|
||||
TCP retransmissions reuse the same sequence number, so deduplication by sequence number prevents duplicate data in the reassembled stream. This is a simplified reassembly that does not handle overlapping segments (where retransmissions contain different data), but it covers the common case.
|
||||
|
||||
### Protocol Identification
|
||||
|
||||
The `identify_protocol` function in `network/protocols.py` performs Deep Packet Inspection using byte prefix matching:
|
||||
|
||||
```python
|
||||
def identify_protocol(payload: bytes) -> str:
|
||||
if not payload:
|
||||
return "unknown"
|
||||
|
||||
if _is_http_request(payload):
|
||||
return "http"
|
||||
if payload.startswith(HTTP_RESPONSE_PREFIX):
|
||||
return "http"
|
||||
if (len(payload) > 2
|
||||
and payload[:2] == TLS_RECORD_PREFIX):
|
||||
return "tls"
|
||||
if payload.startswith(SSH_PREFIX):
|
||||
return "ssh"
|
||||
if payload.startswith(SMTP_BANNER_PREFIX):
|
||||
return "smtp"
|
||||
|
||||
return "unknown"
|
||||
```
|
||||
|
||||
HTTP requests are identified by checking if the first word before a space is a known HTTP method (`GET`, `POST`, `PUT`, etc.). TLS records start with `\x16\x03` (ContentType=Handshake + major version 3). SSH banners start with `SSH-`. SMTP server greetings start with `220`.
|
||||
|
||||
This matters for DLP because the same sensitive data requires different handling depending on the transport protocol. An SSN in an HTTP body can be read and flagged with high confidence. The same SSN in a TLS-encrypted stream cannot be read, but you can flag the flow as "encrypted traffic containing unknown data" and correlate with other signals.
|
||||
|
||||
### DNS Exfiltration Detection
|
||||
|
||||
The `DnsExfilDetector` in `network/exfiltration.py` analyzes DNS queries for patterns that suggest data tunneling:
|
||||
|
||||
```python
|
||||
def _check_subdomain_entropy(self, name, src_ip, dst_ip):
|
||||
parts = name.split(".")
|
||||
if len(parts) < 3:
|
||||
return None
|
||||
|
||||
subdomain = ".".join(parts[:-2])
|
||||
if not subdomain:
|
||||
return None
|
||||
|
||||
entropy = shannon_entropy_str(subdomain)
|
||||
if entropy > self._entropy_threshold:
|
||||
return ExfilIndicator(
|
||||
indicator_type="dns_high_entropy",
|
||||
description=(
|
||||
f"High subdomain entropy ({entropy:.2f}) "
|
||||
f"suggesting DNS tunneling"
|
||||
),
|
||||
confidence=min(
|
||||
0.95,
|
||||
0.50 + (entropy - 3.0) * 0.15,
|
||||
),
|
||||
source_ip=src_ip,
|
||||
dst_ip=dst_ip,
|
||||
evidence=name,
|
||||
)
|
||||
```
|
||||
|
||||
Legitimate subdomains (`www`, `mail`, `api`, `cdn`) have very low entropy. A query like `aGVsbG8gd29ybGQ.evil.com` has subdomain entropy above 4.0 because the base64-encoded data uses most of the alphanumeric character space. The detector extracts everything before the last two domain labels (the registerable domain), computes Shannon entropy, and flags queries above the threshold.
|
||||
|
||||
The confidence score scales linearly from 0.50 (at entropy 3.0) to 0.95 (at entropy 6.0). This captures the observation that higher entropy means more confident detection: entropy 4.1 might be a CDN hash, but entropy 5.5 is almost certainly encoded data.
|
||||
|
||||
## Compliance and Severity Classification
|
||||
|
||||
### Severity Mapping
|
||||
|
||||
The `score_to_severity` function in `compliance.py` maps confidence scores to severity levels using a threshold table:
|
||||
|
||||
```python
|
||||
SEVERITY_SCORE_THRESHOLDS = [
|
||||
(0.85, "critical"),
|
||||
(0.65, "high"),
|
||||
(0.40, "medium"),
|
||||
(0.20, "low"),
|
||||
]
|
||||
|
||||
def score_to_severity(score: float) -> Severity:
|
||||
for threshold, severity in SEVERITY_SCORE_THRESHOLDS:
|
||||
if score >= threshold:
|
||||
return severity
|
||||
return "low"
|
||||
```
|
||||
|
||||
The thresholds are tuned so that:
|
||||
- **Critical** (0.85+): checksum-validated matches with context keywords (e.g., SSN near "social security")
|
||||
- **High** (0.65+): checksum-validated matches or strong context without validation
|
||||
- **Medium** (0.40+): pattern matches without strong validation or context
|
||||
- **Low** (0.20+): weak matches that might be false positives
|
||||
|
||||
### Framework Mapping
|
||||
|
||||
The `RULE_FRAMEWORK_MAP` in `compliance.py` is a static lookup table:
|
||||
|
||||
```python
|
||||
RULE_FRAMEWORK_MAP = {
|
||||
"PII_SSN": ["HIPAA", "CCPA", "GLBA", "GDPR"],
|
||||
"FIN_CREDIT_CARD": ["PCI_DSS", "GLBA"],
|
||||
"FIN_IBAN": ["GDPR", "GLBA"],
|
||||
"HEALTH_MEDICAL_RECORD": ["HIPAA"],
|
||||
...
|
||||
}
|
||||
```
|
||||
|
||||
Each rule maps to the compliance frameworks that regulate that data type. SSNs trigger four frameworks because they are considered protected health information (HIPAA), personal information (CCPA), financial identifiers (GLBA), and personal data (GDPR). Credit card PANs only trigger PCI-DSS and GLBA because HIPAA and GDPR do not specifically regulate financial card numbers.
|
||||
|
||||
The mapping is intentionally conservative. An SSN could trigger SOX if it appears in financial reporting data, but without business context the scanner cannot determine that. The listed frameworks are the ones where the mere presence of the data type creates a compliance obligation.
|
||||
|
||||
## Redaction
|
||||
|
||||
The `redact` function in `redaction.py` builds a snippet with masked content:
|
||||
|
||||
```python
|
||||
def redact(text, start, end, style="partial"):
|
||||
matched = text[start:end]
|
||||
|
||||
if style == "none":
|
||||
return _build_snippet(text, start, end, matched)
|
||||
if style == "full":
|
||||
return _build_snippet(
|
||||
text, start, end, REDACTED_LABEL
|
||||
)
|
||||
|
||||
redacted = _partial_redact(matched)
|
||||
return _build_snippet(text, start, end, redacted)
|
||||
```
|
||||
|
||||
The `_partial_redact` function applies format-aware masking:
|
||||
|
||||
```python
|
||||
def _partial_redact(value):
|
||||
stripped = value.replace("-", "").replace(" ", "")
|
||||
|
||||
if len(stripped) >= 9 and stripped.isdigit():
|
||||
return MASK_CHAR * (len(value) - 4) + value[-4:]
|
||||
|
||||
if "@" in value:
|
||||
local, domain = value.rsplit("@", maxsplit=1)
|
||||
masked_local = (
|
||||
local[0] + MASK_CHAR * (len(local) - 1)
|
||||
)
|
||||
return f"{masked_local}@{domain}"
|
||||
|
||||
if len(value) > 8:
|
||||
visible = max(4, len(value) // 4)
|
||||
return (
|
||||
MASK_CHAR * (len(value) - visible)
|
||||
+ value[-visible:]
|
||||
)
|
||||
|
||||
return MASK_CHAR * len(value)
|
||||
```
|
||||
|
||||
For digit sequences (SSNs, credit cards), it preserves the last 4 digits: `***-**-6789`. For emails, it keeps the first character and domain: `j****@example.com`. For other strings (API keys, tokens), it shows the last 25%. Short values under 8 characters are fully masked.
|
||||
|
||||
The `_build_snippet` function adds 20 characters of context on each side and prepends/appends `...` when the context is truncated. This gives analysts enough surrounding text to understand what the data was near without exposing full document contents.
|
||||
|
||||
## CLI Integration
|
||||
|
||||
### Global Option Propagation
|
||||
|
||||
The Typer callback stores global options in Click's context dict:
|
||||
|
||||
```python
|
||||
@app.callback()
|
||||
def main(ctx: typer.Context, config: ..., verbose: ..., version: ...):
|
||||
ctx.ensure_object(dict)
|
||||
ctx.obj["config_path"] = config
|
||||
ctx.obj["verbose"] = verbose
|
||||
```
|
||||
|
||||
Subcommands retrieve these via `ctx.ensure_object(dict)`:
|
||||
|
||||
```python
|
||||
def _run_scan(ctx, scan_type, target, output_format, output_file):
|
||||
obj: dict[str, Any] = ctx.ensure_object(dict)
|
||||
config_path = obj.get("config_path", "")
|
||||
verbose = obj.get("verbose", False)
|
||||
```
|
||||
|
||||
This pattern lets `dlp-scan -v -c custom.yml file ./data` propagate the verbose flag and config path to the file scan command without duplicating those options on every subcommand.
|
||||
|
||||
### Logging Strategy
|
||||
|
||||
The logging level adapts to the output format:
|
||||
|
||||
```python
|
||||
if verbose:
|
||||
configure_logging(level="DEBUG")
|
||||
elif output_format == "console":
|
||||
configure_logging(level="INFO")
|
||||
else:
|
||||
configure_logging(level="WARNING")
|
||||
```
|
||||
|
||||
When output is machine-readable (JSON, SARIF, CSV), logging is set to WARNING so that structlog messages written to stderr do not contaminate stdout. This prevents `dlp-scan file ./data -f json | jq` from breaking because log lines mixed into the JSON output. For console output, INFO-level logging provides progress feedback. Verbose mode enables DEBUG for troubleshooting.
|
||||
|
||||
### Report Conversion
|
||||
|
||||
The `report convert` command reads a JSON scan result and regenerates it in another format:
|
||||
|
||||
```python
|
||||
@report_app.command("convert")
|
||||
def convert(input_file, output_format="sarif", output_file=""):
|
||||
raw = path.read_bytes()
|
||||
data = orjson.loads(raw)
|
||||
result = _rebuild_result(data)
|
||||
|
||||
config = ScanConfig()
|
||||
engine = ScanEngine(config)
|
||||
|
||||
output = engine.generate_report(result, fmt)
|
||||
...
|
||||
```
|
||||
|
||||
The `_rebuild_result` function deserializes the JSON structure back into `ScanResult`, `Finding`, and `Location` objects. It reads from the `scan_metadata` section for scan-level fields and iterates `findings` to reconstruct each `Finding` with its `Location`. This is necessary because `orjson.loads` produces plain dicts, but the reporters expect typed dataclass instances.
|
||||
|
||||
## Testing Strategy
|
||||
|
||||
### Property-Based Testing
|
||||
|
||||
The project uses Hypothesis for property-based testing of detection rules. Instead of testing a few known inputs, Hypothesis generates random strings constrained by rule formats and verifies that the detection pipeline handles them correctly.
|
||||
|
||||
For validators: Hypothesis generates random digit sequences and verifies that `luhn_check`, `iban_check`, and `nhs_check` only return True for inputs that satisfy the mathematical properties (divisibility by 10, mod 97 = 1, mod 11 check digit match).
|
||||
|
||||
For the context boost: Hypothesis generates random text with embedded keywords at varying distances and verifies that the boost is always between 0 and `CONTEXT_BOOST_MAX`, and that closer keywords produce higher boosts.
|
||||
|
||||
### Running Tests
|
||||
|
||||
```bash
|
||||
uv run pytest -m unit # fast unit tests
|
||||
uv run pytest -m integration # tests with file I/O
|
||||
uv run pytest --cov=src # coverage report
|
||||
```
|
||||
|
||||
The test suite uses markers (`unit`, `integration`, `slow`) to separate fast tests from those requiring real filesystem access. The `conftest.py` provides shared fixtures for temporary directories, sample configs, and test data files.
|
||||
|
||||
## Dependencies
|
||||
|
||||
- **typer**: CLI framework with type-hint argument declaration. The `Annotated` style avoids decorators stacking up.
|
||||
- **rich**: Terminal tables with colors. Used by `ConsoleReporter` for severity-colored output.
|
||||
- **structlog**: Structured logging with stdlib integration. JSON or console rendering based on config.
|
||||
- **pydantic**: Config validation. Catches invalid YAML values before the scan starts.
|
||||
- **orjson**: Fast JSON serialization. 3-10x faster than stdlib json for large finding lists.
|
||||
- **ruamel.yaml**: YAML parser that handles 1.2 spec and preserves comments.
|
||||
- **dpkt**: PCAP parsing. ~100x faster than Scapy for bulk packet iteration.
|
||||
- **pymupdf**: PDF text extraction with layout preservation.
|
||||
- **python-docx/openpyxl/xlrd**: Office format extraction.
|
||||
- **defusedxml/lxml**: Safe XML parsing (defusedxml blocks XXE attacks).
|
||||
- **pyarrow/fastavro**: Columnar format extraction (Parquet, Avro).
|
||||
- **asyncpg/aiomysql/pymongo/aiosqlite**: Async database drivers.
|
||||
|
||||
## Next Steps
|
||||
|
||||
You have seen how the code works. Now:
|
||||
1. Try the challenges in [04-CHALLENGES.md](./04-CHALLENGES.md) for extension ideas
|
||||
2. Modify a detection rule and run the tests to see how the scoring changes
|
||||
3. Scan your own files with `dlp-scan file ./your-directory` and inspect the output
|
||||
|
|
@ -0,0 +1,422 @@
|
|||
# 04-CHALLENGES.md
|
||||
|
||||
# Extension Challenges
|
||||
|
||||
You have built a DLP scanner with file, database, and network scanning, a confidence scoring pipeline, compliance mapping, and multi-format reporting. These challenges extend it into new territory.
|
||||
|
||||
Ordered by difficulty. The easy ones take an hour or two. The advanced ones are multi-day efforts that teach you skills used in production DLP systems.
|
||||
|
||||
## Easy Challenges
|
||||
|
||||
### Challenge 1: Add a New PII Rule (Date of Birth)
|
||||
|
||||
**What to build:** A detection rule for dates of birth in common formats: `MM/DD/YYYY`, `YYYY-MM-DD`, `DD-Mon-YYYY`.
|
||||
|
||||
**Why it matters:** Date of birth is classified as PHI under HIPAA's 18 identifiers and as personal data under GDPR. The 2015 Anthem breach exposed 78.8 million records including DOBs, and the combination of DOB + name + zip code is enough to uniquely identify 87% of the US population (Latanya Sweeney's research at Carnegie Mellon).
|
||||
|
||||
**What you will learn:**
|
||||
- Writing regex patterns that match multiple date formats
|
||||
- Adding a validation function that rejects impossible dates (month 13, day 32, Feb 30)
|
||||
- Tuning base_score relative to false positive rate (dates appear everywhere)
|
||||
|
||||
**Hints:**
|
||||
- Create the rule in a new file `detectors/rules/pii_extended.py` and add the rules list to `ALL_RULES` in `registry.py`
|
||||
- Use a low base_score (0.10-0.15) because date strings are extremely common
|
||||
- Context keywords like "date of birth", "dob", "birthday", "born on" should provide the majority of the signal
|
||||
- The validator should parse the matched string into a real date and reject invalid ones
|
||||
- Add the rule to `RULE_FRAMEWORK_MAP` in `compliance.py` with HIPAA and GDPR
|
||||
|
||||
**Test it works:** Create a text file with "Patient DOB: 03/15/1987" and "Order date: 03/15/1987". The first should score higher than the second due to context keywords.
|
||||
|
||||
### Challenge 2: HTML Report Output
|
||||
|
||||
**What to build:** A new reporter that generates a standalone HTML file with a sortable findings table, severity color coding, and a summary chart.
|
||||
|
||||
**Why it matters:** Compliance teams often need to share scan results with non-technical stakeholders who do not have command-line tools. An HTML report that opens in a browser is more accessible than JSON or CSV.
|
||||
|
||||
**What you will learn:**
|
||||
- Implementing the reporter pattern (match the existing protocol)
|
||||
- HTML template generation in Python (string templates or Jinja2)
|
||||
- Adding a new output format to the CLI without modifying existing code
|
||||
|
||||
**Hints:**
|
||||
- Create `reporters/html_report.py` with a `HtmlReporter` class
|
||||
- Add `"html"` to `REPORTER_MAP` in `engine.py` and `VALID_FORMATS` in `commands/scan.py`
|
||||
- Use inline CSS so the report is a single self-contained HTML file with no external dependencies
|
||||
- Color severity levels using the same scheme as the console reporter (red for critical, yellow for medium, green for low)
|
||||
- Include a summary section at the top with counts by severity and framework
|
||||
|
||||
**Test it works:** Run `dlp-scan file ./data -f html -o report.html` and open the file in a browser. The table should be sortable by clicking column headers (add minimal JavaScript for this).
|
||||
|
||||
### Challenge 3: Allowlist by File Path Pattern
|
||||
|
||||
**What to build:** Extend the allowlist system to suppress findings from files matching glob patterns. Currently, `allowlists.file_patterns` exists in the config but is not enforced during scanning.
|
||||
|
||||
**Why it matters:** Test fixtures, mock data, and seed files intentionally contain fake PII. Teams waste hours triaging findings from `tests/fixtures/sample_data.csv` that contain test credit card numbers. Path-based allowlisting eliminates this noise.
|
||||
|
||||
**What you will learn:**
|
||||
- Connecting config to scan-time behavior
|
||||
- Glob pattern matching with `fnmatch`
|
||||
- The difference between value-level and file-level suppression
|
||||
|
||||
**Hints:**
|
||||
- The `AllowlistConfig.file_patterns` field already exists in `config.py`
|
||||
- Add the check in `FileScanner._scan_file` before running detection, or in `_scan_directory` before scanning the file
|
||||
- Match against the relative path from the scan target, not the absolute path
|
||||
- Patterns like `test_*`, `*_fixture*`, and `mock_*` should match filenames
|
||||
|
||||
**Test it works:** Create a file `test_data.txt` with a valid SSN. Scan with `file_patterns: ["test_*"]` in config. The SSN should not appear in results.
|
||||
|
||||
## Intermediate Challenges
|
||||
|
||||
### Challenge 4: Incremental Scanning with Hash Cache
|
||||
|
||||
**What to build:** A scan cache that stores SHA-256 hashes of scanned files and skips unchanged files on subsequent scans.
|
||||
|
||||
**Why it matters:** Large codebases and file shares contain millions of files. Re-scanning unchanged files wastes time. Symantec DLP and Microsoft Purview both use content hashing to skip unchanged files, reducing scan time by 60-90% on repeated scans.
|
||||
|
||||
**What you will learn:**
|
||||
- Content-addressable caching strategies
|
||||
- SQLite as an embedded metadata store
|
||||
- Cache invalidation (the hardest problem in computer science, and one you actually have to solve)
|
||||
|
||||
**Implementation approach:**
|
||||
|
||||
1. **Create `cache.py`** with a `ScanCache` class backed by SQLite
|
||||
- Table: `(file_path TEXT, content_hash TEXT, scan_time TEXT, finding_count INTEGER)`
|
||||
- Hash computation: SHA-256 of file contents
|
||||
- Lookup: if the file exists in cache with the same hash, skip scanning and load cached finding count
|
||||
|
||||
2. **Integrate with `FileScanner`**
|
||||
- Before extracting text, check the cache
|
||||
- After scanning, store the hash and finding count
|
||||
- Add `--no-cache` flag to force full rescan
|
||||
|
||||
3. **Handle invalidation edge cases:**
|
||||
- What if detection rules change between scans? (The same file might produce different findings with new rules)
|
||||
- What if the config changes min_confidence? (Previously-suppressed findings might now be reportable)
|
||||
- What if a file is deleted? (Stale cache entries should not appear in results)
|
||||
|
||||
**Hints:**
|
||||
- Store a hash of the active rule set and config in the cache. If either changes, invalidate the entire cache
|
||||
- Use `aiosqlite` to match the async pattern of the database scanner, or use synchronous sqlite3 since file scanning is already synchronous
|
||||
- The cache file should live next to the config: `.dlp-scanner-cache.db`
|
||||
|
||||
**Extra credit:** Add `dlp-scan cache stats` and `dlp-scan cache clear` subcommands.
|
||||
|
||||
### Challenge 5: Severity Override by Compliance Framework
|
||||
|
||||
**What to build:** A config option that overrides severity based on compliance framework requirements. For example, any PCI-DSS finding should be at least "high" regardless of confidence score, because PCI-DSS does not have a concept of "low severity" unencrypted card data.
|
||||
|
||||
**Why it matters:** Different compliance frameworks have different severity thresholds. GDPR treats unencrypted email addresses as medium priority for remediation, but PCI-DSS treats any unencrypted PAN as a blocking finding. Production DLP tools let compliance teams configure per-framework severity floors.
|
||||
|
||||
**What you will learn:**
|
||||
- Adding config-driven behavior to the scoring pipeline
|
||||
- The tension between confidence-based and policy-based severity
|
||||
- How production DLP tools balance detection accuracy with compliance requirements
|
||||
|
||||
**Implementation approach:**
|
||||
|
||||
1. **Add to config:**
|
||||
```yaml
|
||||
compliance:
|
||||
severity_overrides:
|
||||
PCI_DSS: "high"
|
||||
HIPAA: "medium"
|
||||
```
|
||||
|
||||
2. **Apply after scoring:** In the `_match_to_finding` function (or equivalent), after computing severity from confidence, check if any of the finding's compliance frameworks have a severity floor, and upgrade if necessary
|
||||
|
||||
3. **Preserve original confidence:** The confidence score should not change. Only the severity classification changes. This lets analysts see that a finding scored 0.35 (normally "low") but was elevated to "high" because of PCI-DSS policy
|
||||
|
||||
**Hints:**
|
||||
- Add `severity_overrides: dict[str, str]` to `ComplianceConfig` in `config.py`
|
||||
- Use `SEVERITY_ORDER` from `constants.py` to compare severity levels numerically
|
||||
- Log when a severity is overridden so analysts understand why a low-confidence finding shows up as high severity
|
||||
|
||||
### Challenge 6: Database Column Name Heuristic Scoring
|
||||
|
||||
**What to build:** A pre-scan heuristic that boosts detection confidence for columns whose names suggest sensitive data (e.g., `ssn`, `credit_card_number`, `patient_dob`).
|
||||
|
||||
**Why it matters:** Database schema names are strong metadata signals. A column named `ssn` in a table named `employees` is almost certainly storing Social Security Numbers, even before you look at the data. The Capital One breach investigation found that the compromised S3 bucket contained CSV exports with column headers like `SSN` and `AccountNumber`, which would have been trivially detectable with column-name analysis.
|
||||
|
||||
**What you will learn:**
|
||||
- Schema introspection as a detection signal
|
||||
- Combining metadata and content signals
|
||||
- How production DLP tools use schema analysis to prioritize scanning
|
||||
|
||||
**Implementation approach:**
|
||||
|
||||
1. **Create a column name classifier** with patterns mapping column names to rule IDs:
|
||||
```
|
||||
*ssn*, *social_sec* -> PII_SSN
|
||||
*credit_card*, *card_num*, *pan* -> FIN_CREDIT_CARD
|
||||
*email*, *e_mail* -> PII_EMAIL
|
||||
*dob*, *date_of_birth*, *birthday* -> PII_DOB
|
||||
```
|
||||
|
||||
2. **Apply as a context boost** in the database scanner: when a column name matches a pattern, add a pre-boost to the base score before running the normal detection pipeline
|
||||
|
||||
3. **Carry through to findings:** Add the column name match as additional evidence in the finding's metadata
|
||||
|
||||
**Hints:**
|
||||
- Implement this in `scanners/db_scanner.py` before the detection loop
|
||||
- Use `fnmatch` for column name pattern matching (same as rule filtering)
|
||||
- A modest boost (+0.15 to +0.25) is appropriate. Column names are strong signals but not definitive (a column named `ssn_backup_old` might be empty or encrypted)
|
||||
|
||||
## Advanced Challenges
|
||||
|
||||
### Challenge 7: Custom Rule Language
|
||||
|
||||
**What to build:** A YAML-based rule definition format that lets users create detection rules without writing Python. Rules should support regex patterns, base scores, context keywords, and compliance framework tags.
|
||||
|
||||
**Why it matters:** Production DLP tools (Symantec DLP, Netskope) let compliance teams define custom rules through policy editors because not every regulated data type is covered by built-in rules. European IBANs, Brazilian CPFs, Indian Aadhaar numbers, and industry-specific identifiers all need custom patterns.
|
||||
|
||||
**What you will learn:**
|
||||
- DSL design (keeping it simple enough to be useful, complex enough to be powerful)
|
||||
- Safe regex compilation (preventing ReDoS)
|
||||
- Hot reloading user-defined rules
|
||||
|
||||
**Implementation approach:**
|
||||
|
||||
1. **Define the rule schema:**
|
||||
```yaml
|
||||
rules:
|
||||
- id: CUSTOM_BR_CPF
|
||||
name: "Brazilian CPF Number"
|
||||
pattern: '\b\d{3}\.\d{3}\.\d{3}-\d{2}\b'
|
||||
base_score: 0.40
|
||||
context_keywords: ["cpf", "cadastro"]
|
||||
compliance: ["LGPD"]
|
||||
validator: "mod11"
|
||||
```
|
||||
|
||||
2. **Build a rule loader** that reads YAML files from a `rules/` directory, compiles regex patterns safely (with timeout protection against catastrophic backtracking), and creates `DetectionRule` objects
|
||||
|
||||
3. **Register custom rules** alongside built-in rules in the `DetectorRegistry`
|
||||
|
||||
4. **Add built-in validator references** (mod11, luhn, mod97) that users can reference by name instead of writing Python
|
||||
|
||||
**Gotchas:**
|
||||
- Regex compilation must be safe: a user-provided pattern like `(a+)+b` causes catastrophic backtracking. Consider using the `regex` library with timeout, or validate patterns against known ReDoS patterns
|
||||
- Custom rules should not be able to override or shadow built-in rules. Use ID prefixes (`CUSTOM_`) to namespace them
|
||||
- Validator functions referenced by name need a registry of their own
|
||||
|
||||
### Challenge 8: Real-Time File Monitoring
|
||||
|
||||
**What to build:** A watch mode that monitors directories for file changes using filesystem events and scans new or modified files automatically.
|
||||
|
||||
**Why it matters:** Batch scanning finds problems after the fact. Real-time monitoring catches sensitive data as soon as it hits disk. This is how endpoint DLP agents (CrowdStrike Falcon DLP, Digital Guardian) work: they hook filesystem events and scan in real time.
|
||||
|
||||
**What you will learn:**
|
||||
- Filesystem event monitoring with `watchdog` or `inotify`
|
||||
- Event debouncing (a single file save can trigger multiple events)
|
||||
- Background scanning without blocking the event loop
|
||||
|
||||
**Architecture changes:**
|
||||
|
||||
```
|
||||
┌──────────────────────────────┐
|
||||
│ FileSystemEventHandler │
|
||||
│ (watchdog or inotify) │
|
||||
│ │
|
||||
│ on_modified -> debounce │
|
||||
│ on_created -> scan_file │
|
||||
│ on_moved -> scan_dest │
|
||||
└──────────────┬───────────────┘
|
||||
│
|
||||
▼
|
||||
┌──────────────────────────────┐
|
||||
│ ScanQueue (asyncio.Queue) │
|
||||
│ │
|
||||
│ Dedup by path │
|
||||
│ Rate limit scanning │
|
||||
└──────────────┬───────────────┘
|
||||
│
|
||||
▼
|
||||
┌──────────────────────────────┐
|
||||
│ FileScanner.scan(file) │
|
||||
│ → Finding → Alert │
|
||||
└──────────────────────────────┘
|
||||
```
|
||||
|
||||
**Implementation steps:**
|
||||
|
||||
1. Add `watchdog` as a dependency
|
||||
2. Create `commands/watch.py` with a `dlp-scan watch ./directory` command
|
||||
3. Implement a debouncer that batches filesystem events within a 500ms window
|
||||
4. Use the existing `FileScanner._scan_file` for individual file scanning
|
||||
5. Output findings to console in real time (stream mode, not batch)
|
||||
|
||||
**Gotchas:**
|
||||
- Editor save operations often create temporary files, write to them, then rename. This generates create, modify, and rename events. You need to scan the final file, not the intermediate temp files
|
||||
- Large file copies trigger `on_modified` repeatedly as data is written. Debounce by waiting until the file size stabilizes
|
||||
- The watch mode should respect the same exclude patterns and extension filters as batch scanning
|
||||
|
||||
### Challenge 9: SIEM Integration via Syslog
|
||||
|
||||
**What to build:** A reporter that sends findings to a SIEM (Splunk, Elastic, QRadar) via syslog (RFC 5424) or HTTP Event Collector (Splunk HEC).
|
||||
|
||||
**Why it matters:** DLP findings are useless if they sit in a JSON file that nobody reads. Production DLP deployments send alerts to SIEMs where SOC analysts triage them alongside firewall logs, EDR alerts, and authentication events. Correlating a DLP finding with a VPN login from an unusual location turns a medium-severity alert into an incident.
|
||||
|
||||
**What you will learn:**
|
||||
- Syslog protocol formatting (RFC 5424 structured data)
|
||||
- HTTP-based log shipping (Splunk HEC, Elastic Ingest)
|
||||
- Alert fatigue management (batching, deduplication, severity filtering)
|
||||
|
||||
**Implementation approach:**
|
||||
|
||||
1. **Create `reporters/syslog_reporter.py`** that formats findings as RFC 5424 syslog messages:
|
||||
```
|
||||
<134>1 2026-04-08T10:30:00Z scanner dlp-scan - -
|
||||
[finding@dlp rule_id="PII_SSN" severity="critical"
|
||||
confidence="0.92" uri="employees.csv"] SSN detected
|
||||
```
|
||||
|
||||
2. **Add Splunk HEC support** as an alternative transport: POST JSON payloads to `https://splunk:8088/services/collector/event` with an HEC token
|
||||
|
||||
3. **Add config:**
|
||||
```yaml
|
||||
output:
|
||||
siem:
|
||||
type: "syslog" # or "splunk_hec"
|
||||
host: "siem.corp.com"
|
||||
port: 514
|
||||
protocol: "tcp" # or "udp"
|
||||
hec_token: "" # for Splunk HEC
|
||||
```
|
||||
|
||||
4. **Implement batching:** Send findings in batches of 50 with a 5-second flush interval to avoid overwhelming the SIEM
|
||||
|
||||
## Expert Challenges
|
||||
|
||||
### Challenge 10: Machine Learning False Positive Reduction
|
||||
|
||||
**What to build:** A feedback loop where analysts can mark findings as true positive or false positive, and a classifier learns to suppress likely false positives on future scans.
|
||||
|
||||
**Why it matters:** The single biggest complaint about DLP tools is false positive volume. Symantec DLP deployments commonly see 40-60% false positive rates on initial rollout. Analysts spend hours dismissing findings that match SSN patterns but are actually serial numbers, batch IDs, or zip+4 codes. A classifier trained on analyst feedback can reduce false positives by 70-80% while maintaining detection recall.
|
||||
|
||||
**What you will learn:**
|
||||
- Feature engineering from detection signals (confidence, context keywords found, rule type, file type, surrounding text patterns)
|
||||
- Online learning: updating a model as new feedback arrives without retraining from scratch
|
||||
- The precision-recall tradeoff in security tooling (a false negative is a missed breach; a false positive is analyst fatigue)
|
||||
|
||||
**Implementation phases:**
|
||||
|
||||
**Phase 1: Feedback Collection**
|
||||
- Add `dlp-scan feedback <finding_id> --true-positive` and `--false-positive` commands
|
||||
- Store feedback in a SQLite database: `(finding_id, rule_id, features_json, label, timestamp)`
|
||||
- Extract features: confidence, rule_id, file extension, context keywords matched, co-occurrence count, surrounding text entropy
|
||||
|
||||
**Phase 2: Classifier**
|
||||
- Train a logistic regression or gradient boosted tree on accumulated feedback
|
||||
- Features: one-hot encode rule_id, numeric confidence, boolean context_found, file_extension category
|
||||
- Use scikit-learn with ONNX export for deployment without the full sklearn dependency
|
||||
|
||||
**Phase 3: Integration**
|
||||
- After the detection pipeline produces matches, run the classifier as a post-filter
|
||||
- Matches classified as likely false positives get demoted (severity lowered, or moved to a "suppressed" section)
|
||||
- Never fully suppress a detection. Always show suppressed findings in a separate section so analysts can audit the classifier
|
||||
|
||||
**Success criteria:**
|
||||
- [ ] Feedback collection works and stores features
|
||||
- [ ] Classifier trains on 50+ labeled examples
|
||||
- [ ] False positive rate drops by at least 30% on held-out test set
|
||||
- [ ] No true positives are fully suppressed (only demoted)
|
||||
- [ ] Model retrains automatically when feedback count crosses thresholds (100, 500, 1000)
|
||||
|
||||
## Real-World Integration Challenges
|
||||
|
||||
### Integrate with GitHub Code Scanning
|
||||
|
||||
**The goal:** Upload SARIF output to GitHub Code Scanning so DLP findings appear as annotations on pull requests.
|
||||
|
||||
**What you will learn:**
|
||||
- GitHub Code Scanning API
|
||||
- SARIF upload via GitHub Actions
|
||||
- CI/CD pipeline integration for security tooling
|
||||
|
||||
**Steps:**
|
||||
|
||||
1. Create a GitHub Actions workflow that runs `dlp-scan file . -f sarif -o results.sarif` on pull requests
|
||||
2. Upload the SARIF file using the `github/codeql-action/upload-sarif` action
|
||||
3. Configure `on: pull_request` to scan only changed files (use `git diff --name-only` to get the list)
|
||||
4. Set severity filtering so only high/critical findings block the PR
|
||||
|
||||
### Scan AWS S3 Buckets
|
||||
|
||||
**The goal:** Add an S3 scanner that lists objects in a bucket, downloads them to a temp directory, and scans with the existing file scanner.
|
||||
|
||||
**What you will learn:**
|
||||
- boto3 integration for S3 object listing and download
|
||||
- Temporary file management for large object scanning
|
||||
- Credential handling (IAM roles vs. access keys)
|
||||
|
||||
**Steps:**
|
||||
|
||||
1. Add `dlp-scan s3 s3://bucket-name/prefix` command
|
||||
2. Use boto3 to list objects, filter by extension
|
||||
3. Download each object to a temp directory (use `tempfile.mkdtemp`)
|
||||
4. Scan with `FileScanner` and map findings back to S3 URIs
|
||||
5. Clean up temp files after scanning
|
||||
|
||||
This directly addresses the Capital One breach scenario: unencrypted PII in S3 buckets that nobody knew existed.
|
||||
|
||||
## Performance Challenge
|
||||
|
||||
### Handle 1 Million Files
|
||||
|
||||
**The goal:** Make the file scanner handle a directory with 1 million files without running out of memory or taking more than an hour.
|
||||
|
||||
**Current bottleneck:** `Path.rglob("*")` generates a list of all files before scanning starts. With 1 million files, this consumes significant memory and delays the first scan result.
|
||||
|
||||
**Optimization approaches:**
|
||||
|
||||
**Approach 1: Streaming directory walk**
|
||||
- Replace `rglob` with `os.scandir` recursive walk that yields files one at a time
|
||||
- Process and discard each file before reading the next
|
||||
- Memory stays constant regardless of directory size
|
||||
|
||||
**Approach 2: Parallel extraction**
|
||||
- Use `concurrent.futures.ThreadPoolExecutor` for I/O-bound extraction (file reads, PDF parsing)
|
||||
- Use `concurrent.futures.ProcessPoolExecutor` for CPU-bound detection (regex matching on large texts)
|
||||
- Tune pool sizes based on profiling
|
||||
|
||||
**Approach 3: Prioritized scanning**
|
||||
- Scan high-risk extensions first (`.csv`, `.xlsx`, `.sql`) before low-risk ones (`.log`, `.txt`)
|
||||
- Report findings as they are discovered (streaming output) instead of waiting for the full scan to complete
|
||||
|
||||
**Benchmark it:**
|
||||
|
||||
```bash
|
||||
time dlp-scan file /large-directory -f json -o results.json
|
||||
```
|
||||
|
||||
Target: under 60 minutes for 1 million files with an average file size of 10KB.
|
||||
|
||||
## Challenge Completion
|
||||
|
||||
Track your progress:
|
||||
|
||||
- [ ] Easy 1: Date of Birth Rule
|
||||
- [ ] Easy 2: HTML Report Output
|
||||
- [ ] Easy 3: Allowlist by File Path
|
||||
- [ ] Intermediate 4: Incremental Scanning
|
||||
- [ ] Intermediate 5: Severity Override
|
||||
- [ ] Intermediate 6: Column Name Heuristic
|
||||
- [ ] Advanced 7: Custom Rule Language
|
||||
- [ ] Advanced 8: Real-Time Monitoring
|
||||
- [ ] Advanced 9: SIEM Integration
|
||||
- [ ] Expert 10: ML False Positive Reduction
|
||||
- [ ] Integration: GitHub Code Scanning
|
||||
- [ ] Integration: S3 Bucket Scanning
|
||||
- [ ] Performance: 1 Million Files
|
||||
|
||||
## Study Real Implementations
|
||||
|
||||
Compare your work to production DLP tools:
|
||||
|
||||
- **Nightfall AI**: Cloud-native DLP with ML-based detection. Open-sourced their detection patterns. Look at how they handle multi-format extraction
|
||||
- **truffleHog**: Focuses on credential detection in git repos. Their entropy-based detection and regex patterns for API keys are similar to this project's credential rules
|
||||
- **detect-secrets**: Yelp's secret scanner. Compare their plugin architecture to the detector registry pattern in this project
|
||||
- **Microsoft Purview**: Enterprise DLP with 300+ built-in sensitive information types. Their documentation on exact data match (EDM) and trainable classifiers shows where the field is heading
|
||||
|
|
@ -0,0 +1,181 @@
|
|||
# ©AngelaMos | 2026
|
||||
# pyproject.toml
|
||||
|
||||
[project]
|
||||
name = "dlp-scanner"
|
||||
version = "0.1.0"
|
||||
description = "Data Loss Prevention scanner for files, databases, and network traffic"
|
||||
requires-python = ">=3.12"
|
||||
dependencies = [
|
||||
"typer>=0.15.0",
|
||||
"rich>=14.0.0",
|
||||
"structlog>=25.0.0",
|
||||
"pydantic>=2.10.0",
|
||||
"orjson>=3.10.0",
|
||||
"ruamel.yaml>=0.18.0",
|
||||
"pymupdf>=1.25.0",
|
||||
"python-docx>=1.1.0",
|
||||
"openpyxl>=3.1.0",
|
||||
"xlrd>=2.0.0",
|
||||
"defusedxml>=0.7.0",
|
||||
"lxml>=5.0.0",
|
||||
"pyarrow>=16.0.0",
|
||||
"fastavro>=1.9.0",
|
||||
"extract-msg>=0.50.0",
|
||||
"asyncpg>=0.30.0",
|
||||
"aiomysql>=0.2.0",
|
||||
"pymongo>=4.10.0",
|
||||
"aiosqlite>=0.21.0",
|
||||
"dpkt>=1.9.0",
|
||||
]
|
||||
|
||||
[project.scripts]
|
||||
dlp-scan = "dlp_scanner.cli:app"
|
||||
|
||||
[build-system]
|
||||
requires = ["hatchling"]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[tool.hatch.build.targets.wheel]
|
||||
packages = ["src/dlp_scanner"]
|
||||
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
"ruff>=0.11.0",
|
||||
"mypy>=1.15.0",
|
||||
"yapf>=0.43.0",
|
||||
"pytest>=8.3.0",
|
||||
"pytest-asyncio>=0.25.0",
|
||||
"pytest-cov>=6.0.0",
|
||||
"hypothesis>=6.130.0",
|
||||
]
|
||||
|
||||
[tool.ruff]
|
||||
line-length = 75
|
||||
indent-width = 4
|
||||
target-version = "py312"
|
||||
src = ["src"]
|
||||
|
||||
[tool.ruff.lint]
|
||||
select = [
|
||||
"E",
|
||||
"W",
|
||||
"F",
|
||||
"B",
|
||||
"S",
|
||||
"C90",
|
||||
"N",
|
||||
"UP",
|
||||
"SIM",
|
||||
"PTH",
|
||||
"PERF",
|
||||
"RUF",
|
||||
"PL",
|
||||
"TRY",
|
||||
"LOG",
|
||||
]
|
||||
ignore = [
|
||||
"S101",
|
||||
"S112",
|
||||
"TRY003",
|
||||
"PLR2004",
|
||||
"PLR0913",
|
||||
"PLR0911",
|
||||
"PLC0415",
|
||||
"PTH123",
|
||||
"PERF401",
|
||||
"E501",
|
||||
]
|
||||
|
||||
[tool.ruff.lint.per-file-ignores]
|
||||
"tests/**/*.py" = ["S101", "PLR2004", "S104", "S105", "S106"]
|
||||
"src/dlp_scanner/detectors/rules/**/*.py" = ["E501"]
|
||||
"src/dlp_scanner/scanners/db_scanner.py" = ["S608"]
|
||||
"src/dlp_scanner/network/protocols.py" = ["S110"]
|
||||
"src/dlp_scanner/extractors/structured.py" = ["N817"]
|
||||
|
||||
[tool.ruff.lint.mccabe]
|
||||
max-complexity = 12
|
||||
|
||||
[tool.mypy]
|
||||
python_version = "3.12"
|
||||
strict = true
|
||||
warn_return_any = true
|
||||
warn_unused_configs = true
|
||||
show_error_codes = true
|
||||
show_column_numbers = true
|
||||
pretty = true
|
||||
mypy_path = "src"
|
||||
|
||||
[[tool.mypy.overrides]]
|
||||
module = [
|
||||
"dpkt.*",
|
||||
"extract_msg.*",
|
||||
"fastavro.*",
|
||||
"xlrd.*",
|
||||
"docx.*",
|
||||
"openpyxl.*",
|
||||
"fitz.*",
|
||||
"defusedxml.*",
|
||||
"lxml.*",
|
||||
"aiomysql.*",
|
||||
"pymongo.*",
|
||||
"asyncpg.*",
|
||||
"aiosqlite.*",
|
||||
"pyarrow.*",
|
||||
]
|
||||
ignore_missing_imports = true
|
||||
|
||||
[[tool.mypy.overrides]]
|
||||
module = [
|
||||
"dlp_scanner.extractors.email",
|
||||
"dlp_scanner.extractors.office",
|
||||
"dlp_scanner.extractors.structured",
|
||||
"dlp_scanner.scanners.db_scanner",
|
||||
"dlp_scanner.scanners.network_scanner",
|
||||
"dlp_scanner.network.pcap",
|
||||
]
|
||||
disallow_any_expr = false
|
||||
warn_return_any = false
|
||||
disable_error_code = ["attr-defined", "unused-coroutine", "no-untyped-call", "import-untyped"]
|
||||
|
||||
[[tool.mypy.overrides]]
|
||||
module = "tests.*"
|
||||
disallow_untyped_defs = false
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
testpaths = ["tests"]
|
||||
asyncio_mode = "auto"
|
||||
addopts = [
|
||||
"--strict-markers",
|
||||
"--tb=short",
|
||||
]
|
||||
markers = [
|
||||
"unit: fast unit tests with no I/O",
|
||||
"integration: tests requiring real file system or DB",
|
||||
"slow: long-running tests",
|
||||
]
|
||||
|
||||
[tool.coverage.run]
|
||||
source = ["src"]
|
||||
branch = true
|
||||
omit = [
|
||||
"*/tests/*",
|
||||
"src/dlp_scanner/extractors/pdf.py",
|
||||
"src/dlp_scanner/extractors/office.py",
|
||||
"src/dlp_scanner/extractors/archive.py",
|
||||
"src/dlp_scanner/extractors/email.py",
|
||||
"src/dlp_scanner/network/pcap.py",
|
||||
"src/dlp_scanner/scanners/network_scanner.py",
|
||||
"src/dlp_scanner/scanners/db_scanner.py",
|
||||
"src/dlp_scanner/reporters/base.py",
|
||||
"src/dlp_scanner/scanners/base.py",
|
||||
]
|
||||
|
||||
[tool.coverage.report]
|
||||
exclude_lines = [
|
||||
"pragma: no cover",
|
||||
"if TYPE_CHECKING:",
|
||||
"raise NotImplementedError",
|
||||
"\\.\\.\\.",
|
||||
]
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
__init__.py
|
||||
"""
|
||||
|
||||
__version__ = "0.1.0"
|
||||
|
|
@ -0,0 +1,73 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
cli.py
|
||||
"""
|
||||
|
||||
|
||||
from typing import Annotated
|
||||
|
||||
import typer
|
||||
|
||||
from dlp_scanner import __version__
|
||||
from dlp_scanner.commands.report import report_app
|
||||
from dlp_scanner.commands.scan import register
|
||||
|
||||
|
||||
app = typer.Typer(
|
||||
name = "dlp-scan",
|
||||
help = (
|
||||
"Data Loss Prevention scanner for files, "
|
||||
"databases, and network traffic"
|
||||
),
|
||||
no_args_is_help = True,
|
||||
)
|
||||
|
||||
|
||||
def _version_callback(value: bool) -> None:
|
||||
"""
|
||||
Print version and exit
|
||||
"""
|
||||
if value:
|
||||
typer.echo(f"dlp-scanner {__version__}")
|
||||
raise typer.Exit()
|
||||
|
||||
|
||||
@app.callback()
|
||||
def main(
|
||||
ctx: typer.Context,
|
||||
config: Annotated[
|
||||
str,
|
||||
typer.Option(
|
||||
"--config",
|
||||
"-c",
|
||||
help = "Path to config YAML file",
|
||||
),
|
||||
] = "",
|
||||
verbose: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
"--verbose",
|
||||
"-v",
|
||||
help = "Enable verbose output",
|
||||
),
|
||||
] = False,
|
||||
version: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
"--version",
|
||||
callback = _version_callback,
|
||||
is_eager = True,
|
||||
help = "Show version and exit",
|
||||
),
|
||||
] = False,
|
||||
) -> None:
|
||||
"""
|
||||
DLP Scanner - detect sensitive data across files, databases, and network captures
|
||||
"""
|
||||
ctx.ensure_object(dict)
|
||||
ctx.obj["config_path"] = config
|
||||
ctx.obj["verbose"] = verbose
|
||||
|
||||
|
||||
register(app)
|
||||
app.add_typer(report_app, name = "report")
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
__init__.py
|
||||
"""
|
||||
|
|
@ -0,0 +1,187 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
report.py
|
||||
"""
|
||||
|
||||
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Annotated, Any
|
||||
|
||||
import orjson
|
||||
import typer
|
||||
|
||||
from dlp_scanner.models import (
|
||||
Finding,
|
||||
Location,
|
||||
ScanResult,
|
||||
)
|
||||
|
||||
|
||||
report_app = typer.Typer(help = "Report conversion and summary")
|
||||
|
||||
VALID_FORMATS: frozenset[str] = frozenset(
|
||||
{
|
||||
"console",
|
||||
"json",
|
||||
"sarif",
|
||||
"csv",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@report_app.command("convert")
|
||||
def convert(
|
||||
input_file: Annotated[
|
||||
str,
|
||||
typer.Argument(help = "JSON scan results file"),
|
||||
],
|
||||
output_format: Annotated[
|
||||
str,
|
||||
typer.Option(
|
||||
"--format",
|
||||
"-f",
|
||||
help = "Target format (json, sarif, csv)",
|
||||
),
|
||||
] = "sarif",
|
||||
output_file: Annotated[
|
||||
str,
|
||||
typer.Option(
|
||||
"--output",
|
||||
"-o",
|
||||
help = "Write converted report to file",
|
||||
),
|
||||
] = "",
|
||||
) -> None:
|
||||
"""
|
||||
Convert a JSON scan result to another format
|
||||
"""
|
||||
from dlp_scanner.config import ScanConfig
|
||||
from dlp_scanner.engine import ScanEngine
|
||||
|
||||
if output_format not in VALID_FORMATS:
|
||||
typer.echo(
|
||||
f"Invalid format: {output_format}",
|
||||
err = True,
|
||||
)
|
||||
raise typer.Exit(code = 1)
|
||||
|
||||
path = Path(input_file)
|
||||
if not path.exists():
|
||||
typer.echo(
|
||||
f"File not found: {input_file}",
|
||||
err = True,
|
||||
)
|
||||
raise typer.Exit(code = 1)
|
||||
|
||||
raw = path.read_bytes()
|
||||
data = orjson.loads(raw)
|
||||
result = _rebuild_result(data)
|
||||
|
||||
config = ScanConfig()
|
||||
engine = ScanEngine(config)
|
||||
|
||||
output = engine.generate_report(result, output_format)
|
||||
|
||||
if output_file:
|
||||
Path(output_file).write_text(output)
|
||||
typer.echo(f"Converted report written to "
|
||||
f"{output_file}")
|
||||
else:
|
||||
typer.echo(output)
|
||||
|
||||
|
||||
@report_app.command("summary")
|
||||
def summary(
|
||||
input_file: Annotated[
|
||||
str,
|
||||
typer.Argument(help = "JSON scan results file"),
|
||||
],
|
||||
) -> None:
|
||||
"""
|
||||
Print summary statistics from a scan result file
|
||||
"""
|
||||
path = Path(input_file)
|
||||
if not path.exists():
|
||||
typer.echo(
|
||||
f"File not found: {input_file}",
|
||||
err = True,
|
||||
)
|
||||
raise typer.Exit(code = 1)
|
||||
|
||||
raw = path.read_bytes()
|
||||
data = orjson.loads(raw)
|
||||
result = _rebuild_result(data)
|
||||
|
||||
from dlp_scanner.reporters.console import (
|
||||
ConsoleReporter,
|
||||
)
|
||||
|
||||
reporter = ConsoleReporter()
|
||||
reporter.display(result)
|
||||
|
||||
|
||||
def _rebuild_result(
|
||||
data: dict[str,
|
||||
Any],
|
||||
) -> ScanResult:
|
||||
"""
|
||||
Rebuild a ScanResult from deserialized JSON report
|
||||
"""
|
||||
meta = data.get("scan_metadata", {})
|
||||
result = ScanResult(
|
||||
targets_scanned = meta.get("targets_scanned",
|
||||
0),
|
||||
)
|
||||
result.scan_id = meta.get("scan_id", result.scan_id)
|
||||
|
||||
if meta.get("scan_completed_at"):
|
||||
result.scan_completed_at = (
|
||||
datetime.fromisoformat(meta["scan_completed_at"])
|
||||
)
|
||||
|
||||
result.errors = meta.get("errors", [])
|
||||
|
||||
for f_data in data.get("findings", []):
|
||||
loc_data = f_data.get("location", {})
|
||||
location = Location(
|
||||
source_type = loc_data.get("source_type",
|
||||
"file"),
|
||||
uri = loc_data.get("uri",
|
||||
""),
|
||||
line = loc_data.get("line"),
|
||||
column = loc_data.get("column"),
|
||||
table_name = loc_data.get("table_name"),
|
||||
column_name = loc_data.get("column_name"),
|
||||
)
|
||||
|
||||
finding = Finding(
|
||||
rule_id = f_data.get("rule_id",
|
||||
""),
|
||||
rule_name = f_data.get("rule_name",
|
||||
""),
|
||||
severity = f_data.get("severity",
|
||||
"low"),
|
||||
confidence = f_data.get("confidence",
|
||||
0.0),
|
||||
location = location,
|
||||
redacted_snippet = f_data.get("redacted_snippet",
|
||||
""),
|
||||
compliance_frameworks = f_data.get(
|
||||
"compliance_frameworks",
|
||||
[]
|
||||
),
|
||||
remediation = f_data.get("remediation",
|
||||
""),
|
||||
)
|
||||
|
||||
if f_data.get("finding_id"):
|
||||
finding.finding_id = f_data["finding_id"]
|
||||
if f_data.get("detected_at"):
|
||||
finding.detected_at = (
|
||||
datetime.fromisoformat(f_data["detected_at"])
|
||||
)
|
||||
|
||||
result.findings.append(finding)
|
||||
|
||||
return result
|
||||
|
|
@ -0,0 +1,195 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
scan.py
|
||||
"""
|
||||
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Annotated, Any
|
||||
|
||||
import typer
|
||||
|
||||
|
||||
FORMAT_HELP: str = ("Output format (console, json, sarif, csv)")
|
||||
OUTPUT_HELP: str = "Write report to file"
|
||||
VALID_FORMATS: frozenset[str] = frozenset(
|
||||
{
|
||||
"console",
|
||||
"json",
|
||||
"sarif",
|
||||
"csv",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def scan_file(
|
||||
ctx: typer.Context,
|
||||
target: Annotated[
|
||||
str,
|
||||
typer.Argument(help = "File or directory path"),
|
||||
],
|
||||
output_format: Annotated[
|
||||
str,
|
||||
typer.Option(
|
||||
"--format",
|
||||
"-f",
|
||||
help = FORMAT_HELP,
|
||||
),
|
||||
] = "console",
|
||||
output_file: Annotated[
|
||||
str,
|
||||
typer.Option(
|
||||
"--output",
|
||||
"-o",
|
||||
help = OUTPUT_HELP,
|
||||
),
|
||||
] = "",
|
||||
) -> None:
|
||||
"""
|
||||
Scan files and directories for sensitive data
|
||||
"""
|
||||
_run_scan(ctx, "file", target, output_format, output_file)
|
||||
|
||||
|
||||
def scan_db(
|
||||
ctx: typer.Context,
|
||||
target: Annotated[
|
||||
str,
|
||||
typer.Argument(help = "Database connection URI"),
|
||||
],
|
||||
output_format: Annotated[
|
||||
str,
|
||||
typer.Option(
|
||||
"--format",
|
||||
"-f",
|
||||
help = FORMAT_HELP,
|
||||
),
|
||||
] = "console",
|
||||
output_file: Annotated[
|
||||
str,
|
||||
typer.Option(
|
||||
"--output",
|
||||
"-o",
|
||||
help = OUTPUT_HELP,
|
||||
),
|
||||
] = "",
|
||||
) -> None:
|
||||
"""
|
||||
Scan database tables for sensitive data
|
||||
"""
|
||||
_run_scan(ctx, "db", target, output_format, output_file)
|
||||
|
||||
|
||||
def scan_network(
|
||||
ctx: typer.Context,
|
||||
target: Annotated[
|
||||
str,
|
||||
typer.Argument(help = "PCAP file path"),
|
||||
],
|
||||
output_format: Annotated[
|
||||
str,
|
||||
typer.Option(
|
||||
"--format",
|
||||
"-f",
|
||||
help = FORMAT_HELP,
|
||||
),
|
||||
] = "console",
|
||||
output_file: Annotated[
|
||||
str,
|
||||
typer.Option(
|
||||
"--output",
|
||||
"-o",
|
||||
help = OUTPUT_HELP,
|
||||
),
|
||||
] = "",
|
||||
) -> None:
|
||||
"""
|
||||
Scan network capture files for sensitive data in transit
|
||||
"""
|
||||
_run_scan(
|
||||
ctx,
|
||||
"network",
|
||||
target,
|
||||
output_format,
|
||||
output_file,
|
||||
)
|
||||
|
||||
|
||||
def register(app: typer.Typer) -> None:
|
||||
"""
|
||||
Register scan commands on the root app
|
||||
"""
|
||||
app.command("file")(scan_file)
|
||||
app.command("db")(scan_db)
|
||||
app.command("network")(scan_network)
|
||||
|
||||
|
||||
def _run_scan(
|
||||
ctx: typer.Context,
|
||||
scan_type: str,
|
||||
target: str,
|
||||
output_format: str,
|
||||
output_file: str,
|
||||
) -> None:
|
||||
"""
|
||||
Shared scan execution logic
|
||||
"""
|
||||
from dlp_scanner.config import (
|
||||
ScanConfig,
|
||||
load_config,
|
||||
)
|
||||
from dlp_scanner.engine import ScanEngine
|
||||
from dlp_scanner.log import configure_logging
|
||||
|
||||
if output_format not in VALID_FORMATS:
|
||||
typer.echo(
|
||||
f"Invalid format: {output_format}. "
|
||||
f"Choose from: "
|
||||
f"{', '.join(sorted(VALID_FORMATS))}",
|
||||
err = True,
|
||||
)
|
||||
raise typer.Exit(code = 1)
|
||||
|
||||
obj: dict[str, Any] = ctx.ensure_object(dict)
|
||||
config_path: str = obj.get("config_path", "")
|
||||
verbose: bool = obj.get("verbose", False)
|
||||
|
||||
if verbose:
|
||||
configure_logging(level = "DEBUG")
|
||||
elif output_format == "console":
|
||||
configure_logging(level = "INFO")
|
||||
else:
|
||||
configure_logging(level = "WARNING")
|
||||
|
||||
config: ScanConfig
|
||||
cfg_path = Path(config_path) if config_path else None
|
||||
if cfg_path and cfg_path.exists():
|
||||
config = load_config(cfg_path)
|
||||
else:
|
||||
config = ScanConfig()
|
||||
|
||||
config.output.format = output_format
|
||||
if output_file:
|
||||
config.output.output_file = output_file
|
||||
|
||||
engine = ScanEngine(config)
|
||||
|
||||
scan_methods = {
|
||||
"file": engine.scan_files,
|
||||
"db": engine.scan_database,
|
||||
"network": engine.scan_network,
|
||||
}
|
||||
|
||||
result = scan_methods[scan_type](target)
|
||||
|
||||
if output_file:
|
||||
engine.write_report(result, output_file)
|
||||
typer.echo(f"Report written to {output_file}")
|
||||
elif output_format == "console":
|
||||
engine.display_console(result)
|
||||
else:
|
||||
output = engine.generate_report(result)
|
||||
typer.echo(output)
|
||||
|
||||
if result.errors:
|
||||
raise typer.Exit(code = 1)
|
||||
|
|
@ -0,0 +1,245 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
compliance.py
|
||||
"""
|
||||
|
||||
|
||||
from dlp_scanner.constants import (
|
||||
SEVERITY_SCORE_THRESHOLDS,
|
||||
Severity,
|
||||
)
|
||||
|
||||
|
||||
RULE_FRAMEWORK_MAP: dict[str,
|
||||
list[str]] = {
|
||||
"PII_SSN": ["HIPAA",
|
||||
"CCPA",
|
||||
"GLBA",
|
||||
"GDPR"],
|
||||
"PII_EMAIL": ["GDPR",
|
||||
"CCPA"],
|
||||
"PII_PHONE": ["GDPR",
|
||||
"CCPA",
|
||||
"HIPAA"],
|
||||
"PII_PHONE_INTL": ["GDPR",
|
||||
"CCPA"],
|
||||
"PII_PASSPORT_US": ["GDPR",
|
||||
"CCPA"],
|
||||
"PII_PASSPORT_UK": ["GDPR"],
|
||||
"PII_DRIVERS_LICENSE": ["CCPA",
|
||||
"HIPAA"],
|
||||
"PII_DRIVERS_LICENSE_FL": ["CCPA",
|
||||
"HIPAA"],
|
||||
"PII_DRIVERS_LICENSE_IL": ["CCPA",
|
||||
"HIPAA"],
|
||||
"PII_IPV4": ["GDPR"],
|
||||
"FIN_CREDIT_CARD_VISA": ["PCI_DSS",
|
||||
"GLBA"],
|
||||
"FIN_CREDIT_CARD_MC": ["PCI_DSS",
|
||||
"GLBA"],
|
||||
"FIN_CREDIT_CARD_AMEX": ["PCI_DSS",
|
||||
"GLBA"],
|
||||
"FIN_CREDIT_CARD_DISC": ["PCI_DSS",
|
||||
"GLBA"],
|
||||
"FIN_IBAN": ["GDPR",
|
||||
"GLBA"],
|
||||
"FIN_NHS_NUMBER": ["GDPR"],
|
||||
"CRED_AWS_ACCESS_KEY": [],
|
||||
"CRED_GITHUB_TOKEN": [],
|
||||
"CRED_GITHUB_FINE_GRAINED": [],
|
||||
"CRED_GITHUB_OAUTH": [],
|
||||
"CRED_GITHUB_APP": [],
|
||||
"CRED_JWT": [],
|
||||
"CRED_STRIPE_KEY": [],
|
||||
"CRED_SLACK_TOKEN": [],
|
||||
"CRED_GENERIC_API_KEY": [],
|
||||
"CRED_PRIVATE_KEY": [],
|
||||
"HEALTH_MEDICAL_RECORD": ["HIPAA"],
|
||||
"HEALTH_DEA_NUMBER": ["HIPAA"],
|
||||
"HEALTH_NPI": ["HIPAA"],
|
||||
"NET_HIGH_ENTROPY": [],
|
||||
"NET_DNS_EXFIL_LONG_LABEL": [],
|
||||
"NET_DNS_EXFIL_HIGH_ENTROPY": [],
|
||||
"NET_DNS_EXFIL_LONG_QNAME": [],
|
||||
"NET_DNS_EXFIL_TXT_VOLUME": [],
|
||||
"NET_ENCODED_BASE64": [],
|
||||
"NET_ENCODED_HEX": [],
|
||||
}
|
||||
|
||||
RULE_REMEDIATION_MAP: dict[
|
||||
str,
|
||||
str] = {
|
||||
"PII_SSN": (
|
||||
"Remove or encrypt SSNs. Use tokenization "
|
||||
"for storage. Never store in plaintext."
|
||||
),
|
||||
"PII_EMAIL": (
|
||||
"Evaluate if email storage is necessary. "
|
||||
"Hash or pseudonymize where possible."
|
||||
),
|
||||
"PII_PHONE": (
|
||||
"Restrict access to phone number fields. "
|
||||
"Consider masking in non-production environments."
|
||||
),
|
||||
"PII_PHONE_INTL": (
|
||||
"Restrict access to phone number fields. "
|
||||
"Consider masking in non-production environments."
|
||||
),
|
||||
"PII_PASSPORT_US": (
|
||||
"Passport numbers must be encrypted at rest. "
|
||||
"Limit access to identity verification systems."
|
||||
),
|
||||
"PII_PASSPORT_UK": (
|
||||
"Passport numbers must be encrypted at rest. "
|
||||
"Limit access to identity verification systems."
|
||||
),
|
||||
"PII_IPV4": (
|
||||
"Evaluate whether IP address storage is necessary. "
|
||||
"Anonymize or pseudonymize where possible."
|
||||
),
|
||||
"PII_DRIVERS_LICENSE": (
|
||||
"Encrypt driver's license numbers at rest. "
|
||||
"Restrict access per CCPA/HIPAA requirements."
|
||||
),
|
||||
"PII_DRIVERS_LICENSE_FL": (
|
||||
"Encrypt driver's license numbers at rest. "
|
||||
"Restrict access per CCPA/HIPAA requirements."
|
||||
),
|
||||
"PII_DRIVERS_LICENSE_IL": (
|
||||
"Encrypt driver's license numbers at rest. "
|
||||
"Restrict access per CCPA/HIPAA requirements."
|
||||
),
|
||||
"FIN_CREDIT_CARD_VISA": (
|
||||
"PCI-DSS requires PANs to be encrypted, hashed, "
|
||||
"or truncated. Never store in plaintext."
|
||||
),
|
||||
"FIN_CREDIT_CARD_MC": (
|
||||
"PCI-DSS requires PANs to be encrypted, hashed, "
|
||||
"or truncated. Never store in plaintext."
|
||||
),
|
||||
"FIN_CREDIT_CARD_AMEX": (
|
||||
"PCI-DSS requires PANs to be encrypted, hashed, "
|
||||
"or truncated. Never store in plaintext."
|
||||
),
|
||||
"FIN_CREDIT_CARD_DISC": (
|
||||
"PCI-DSS requires PANs to be encrypted, hashed, "
|
||||
"or truncated. Never store in plaintext."
|
||||
),
|
||||
"FIN_IBAN": (
|
||||
"Encrypt IBAN numbers at rest. "
|
||||
"Restrict access to financial systems."
|
||||
),
|
||||
"FIN_NHS_NUMBER": (
|
||||
"NHS numbers are personal data under UK GDPR. "
|
||||
"Encrypt at rest and restrict access."
|
||||
),
|
||||
"CRED_AWS_ACCESS_KEY": (
|
||||
"Rotate exposed AWS credentials immediately. "
|
||||
"Use IAM roles or Vault dynamic secrets."
|
||||
),
|
||||
"CRED_GITHUB_TOKEN": (
|
||||
"Revoke the token at github.com/settings/tokens. "
|
||||
"Use environment variables, not hardcoded values."
|
||||
),
|
||||
"CRED_GITHUB_FINE_GRAINED": (
|
||||
"Revoke the token at github.com/settings/tokens. "
|
||||
"Use environment variables, not hardcoded values."
|
||||
),
|
||||
"CRED_GITHUB_OAUTH": (
|
||||
"Revoke the OAuth token in GitHub settings. "
|
||||
"Store tokens in a secrets manager."
|
||||
),
|
||||
"CRED_GITHUB_APP": (
|
||||
"Revoke the app installation token. "
|
||||
"Rotate app private keys if compromised."
|
||||
),
|
||||
"CRED_JWT": (
|
||||
"Rotate the signing key if the JWT secret is "
|
||||
"exposed. Never hardcode tokens in source."
|
||||
),
|
||||
"CRED_STRIPE_KEY": (
|
||||
"Rotate the Stripe key at dashboard.stripe.com. "
|
||||
"Use restricted keys with minimal permissions."
|
||||
),
|
||||
"CRED_SLACK_TOKEN": (
|
||||
"Revoke the Slack token in workspace settings. "
|
||||
"Use environment variables for bot tokens."
|
||||
),
|
||||
"CRED_GENERIC_API_KEY": (
|
||||
"Rotate the exposed API key immediately. "
|
||||
"Store secrets in a vault, not in source code."
|
||||
),
|
||||
"CRED_PRIVATE_KEY": (
|
||||
"Rotate the compromised key pair. Store private "
|
||||
"keys in a secrets manager, never in source code."
|
||||
),
|
||||
"HEALTH_MEDICAL_RECORD": (
|
||||
"MRNs are PHI under HIPAA. Encrypt at rest and "
|
||||
"apply minimum necessary access controls."
|
||||
),
|
||||
"HEALTH_DEA_NUMBER": (
|
||||
"DEA numbers identify prescribers of controlled "
|
||||
"substances. Encrypt and restrict access per HIPAA."
|
||||
),
|
||||
"HEALTH_NPI": (
|
||||
"NPIs are provider identifiers under HIPAA. "
|
||||
"Restrict access to authorized systems only."
|
||||
),
|
||||
"NET_HIGH_ENTROPY": (
|
||||
"High entropy data may indicate encrypted or "
|
||||
"compressed secrets in transit. Investigate the flow."
|
||||
),
|
||||
"NET_DNS_EXFIL_LONG_LABEL": (
|
||||
"Unusually long DNS labels may indicate DNS "
|
||||
"tunneling. Investigate the queried domain."
|
||||
),
|
||||
"NET_DNS_EXFIL_HIGH_ENTROPY": (
|
||||
"High-entropy DNS subdomains suggest data "
|
||||
"exfiltration via DNS tunneling. Block the domain."
|
||||
),
|
||||
"NET_DNS_EXFIL_LONG_QNAME": (
|
||||
"Excessively long DNS QNAMEs may carry encoded "
|
||||
"data. Investigate and block suspicious domains."
|
||||
),
|
||||
"NET_DNS_EXFIL_TXT_VOLUME": (
|
||||
"High ratio of TXT queries to a domain suggests "
|
||||
"DNS-based command and control. Investigate traffic."
|
||||
),
|
||||
"NET_ENCODED_BASE64": (
|
||||
"Base64-encoded payloads in network traffic may "
|
||||
"carry exfiltrated data. Inspect the content."
|
||||
),
|
||||
"NET_ENCODED_HEX": (
|
||||
"Hex-encoded payloads in network traffic may "
|
||||
"indicate data exfiltration. Inspect the content."
|
||||
),
|
||||
}
|
||||
|
||||
DEFAULT_REMEDIATION: str = (
|
||||
"Review and restrict access to this data. "
|
||||
"Apply encryption at rest if required by policy."
|
||||
)
|
||||
|
||||
|
||||
def get_frameworks_for_rule(rule_id: str) -> list[str]:
|
||||
"""
|
||||
Return applicable compliance frameworks for a rule
|
||||
"""
|
||||
return RULE_FRAMEWORK_MAP.get(rule_id, [])
|
||||
|
||||
|
||||
def get_remediation_for_rule(rule_id: str) -> str:
|
||||
"""
|
||||
Return remediation guidance for a rule
|
||||
"""
|
||||
return RULE_REMEDIATION_MAP.get(rule_id, DEFAULT_REMEDIATION)
|
||||
|
||||
|
||||
def score_to_severity(score: float) -> Severity:
|
||||
"""
|
||||
Convert a confidence score to a severity level
|
||||
"""
|
||||
for threshold, severity in SEVERITY_SCORE_THRESHOLDS:
|
||||
if score >= threshold:
|
||||
return severity
|
||||
return "low"
|
||||
|
|
@ -0,0 +1,173 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
config.py
|
||||
"""
|
||||
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
from ruamel.yaml import YAML
|
||||
|
||||
from dlp_scanner.constants import (
|
||||
DEFAULT_CONTEXT_WINDOW_TOKENS,
|
||||
DEFAULT_DB_MAX_ROWS,
|
||||
DEFAULT_DB_SAMPLE_PERCENTAGE,
|
||||
DEFAULT_DB_TIMEOUT_SECONDS,
|
||||
DEFAULT_DNS_ENTROPY_THRESHOLD,
|
||||
DEFAULT_ENTROPY_THRESHOLD,
|
||||
DEFAULT_EXCLUDE_PATTERNS,
|
||||
DEFAULT_MAX_FILE_SIZE_MB,
|
||||
DEFAULT_MIN_CONFIDENCE,
|
||||
SCANNABLE_EXTENSIONS,
|
||||
OutputFormat,
|
||||
RedactionStyle,
|
||||
Severity,
|
||||
)
|
||||
|
||||
|
||||
class FileScanConfig(BaseModel):
|
||||
"""
|
||||
Configuration for file scanning
|
||||
"""
|
||||
max_file_size_mb: int = DEFAULT_MAX_FILE_SIZE_MB
|
||||
recursive: bool = True
|
||||
exclude_patterns: list[str] = Field(
|
||||
default_factory = lambda: list(DEFAULT_EXCLUDE_PATTERNS)
|
||||
)
|
||||
include_extensions: list[str] = Field(
|
||||
default_factory = lambda: sorted(SCANNABLE_EXTENSIONS)
|
||||
)
|
||||
|
||||
|
||||
class DatabaseScanConfig(BaseModel):
|
||||
"""
|
||||
Configuration for database scanning
|
||||
"""
|
||||
sample_percentage: int = DEFAULT_DB_SAMPLE_PERCENTAGE
|
||||
max_rows_per_table: int = DEFAULT_DB_MAX_ROWS
|
||||
timeout_seconds: int = DEFAULT_DB_TIMEOUT_SECONDS
|
||||
exclude_tables: list[str] = Field(default_factory = list)
|
||||
include_tables: list[str] = Field(default_factory = list)
|
||||
|
||||
|
||||
class NetworkScanConfig(BaseModel):
|
||||
"""
|
||||
Configuration for network traffic scanning
|
||||
"""
|
||||
bpf_filter: str = ""
|
||||
entropy_threshold: float = DEFAULT_ENTROPY_THRESHOLD
|
||||
dns_label_entropy_threshold: float = (DEFAULT_DNS_ENTROPY_THRESHOLD)
|
||||
max_packets: int = 0
|
||||
|
||||
|
||||
class AllowlistConfig(BaseModel):
|
||||
"""
|
||||
Allowlists for suppressing known false positives
|
||||
"""
|
||||
values: list[str] = Field(default_factory = list)
|
||||
domains: list[str] = Field(default_factory = list)
|
||||
file_patterns: list[str] = Field(default_factory = list)
|
||||
|
||||
|
||||
class DetectionConfig(BaseModel):
|
||||
"""
|
||||
Configuration for detection behavior
|
||||
"""
|
||||
min_confidence: float = DEFAULT_MIN_CONFIDENCE
|
||||
severity_threshold: Severity = "low"
|
||||
context_window_tokens: int = (DEFAULT_CONTEXT_WINDOW_TOKENS)
|
||||
enable_rules: list[str] = Field(default_factory = lambda: ["*"])
|
||||
disable_rules: list[str] = Field(default_factory = list)
|
||||
allowlists: AllowlistConfig = Field(default_factory = AllowlistConfig)
|
||||
|
||||
|
||||
class ComplianceConfig(BaseModel):
|
||||
"""
|
||||
Configuration for compliance framework mapping
|
||||
"""
|
||||
frameworks: list[str] = Field(
|
||||
default_factory = lambda: [
|
||||
"HIPAA",
|
||||
"PCI_DSS",
|
||||
"GDPR",
|
||||
"CCPA",]
|
||||
)
|
||||
|
||||
|
||||
class OutputConfig(BaseModel):
|
||||
"""
|
||||
Configuration for output and reporting
|
||||
"""
|
||||
format: OutputFormat = "console"
|
||||
output_file: str = ""
|
||||
redaction_style: RedactionStyle = "partial"
|
||||
verbose: bool = False
|
||||
color: bool = True
|
||||
|
||||
|
||||
class LoggingConfig(BaseModel):
|
||||
"""
|
||||
Configuration for logging behavior
|
||||
"""
|
||||
level: str = "INFO"
|
||||
json_output: bool = False
|
||||
log_file: str = ""
|
||||
|
||||
|
||||
class ScanConfig(BaseModel):
|
||||
"""
|
||||
Root configuration model for the DLP scanner
|
||||
"""
|
||||
file: FileScanConfig = Field(default_factory = FileScanConfig)
|
||||
database: DatabaseScanConfig = Field(
|
||||
default_factory = DatabaseScanConfig
|
||||
)
|
||||
network: NetworkScanConfig = Field(default_factory = NetworkScanConfig)
|
||||
detection: DetectionConfig = Field(default_factory = DetectionConfig)
|
||||
compliance: ComplianceConfig = Field(
|
||||
default_factory = ComplianceConfig
|
||||
)
|
||||
output: OutputConfig = Field(default_factory = OutputConfig)
|
||||
logging: LoggingConfig = Field(default_factory = LoggingConfig)
|
||||
|
||||
|
||||
def load_config(path: Path | None = None) -> ScanConfig:
|
||||
"""
|
||||
Load configuration from a YAML file or return defaults
|
||||
"""
|
||||
if path is None:
|
||||
candidates = [
|
||||
Path(".dlp-scanner.yml"),
|
||||
Path(".dlp-scanner.yaml"),
|
||||
Path.home() / ".dlp-scanner.yml",
|
||||
]
|
||||
for candidate in candidates:
|
||||
if candidate.exists():
|
||||
path = candidate
|
||||
break
|
||||
|
||||
if path is None or not path.exists():
|
||||
return ScanConfig()
|
||||
|
||||
yaml = YAML(typ = "safe")
|
||||
raw: dict[str, Any] = yaml.load(path) or {}
|
||||
|
||||
scan_section = raw.get("scan", {})
|
||||
return ScanConfig(
|
||||
file = FileScanConfig(**scan_section.get("file",
|
||||
{})),
|
||||
database = DatabaseScanConfig(**scan_section.get("database",
|
||||
{})),
|
||||
network = NetworkScanConfig(**scan_section.get("network",
|
||||
{})),
|
||||
detection = DetectionConfig(**raw.get("detection",
|
||||
{})),
|
||||
compliance = ComplianceConfig(**raw.get("compliance",
|
||||
{})),
|
||||
output = OutputConfig(**raw.get("output",
|
||||
{})),
|
||||
logging = LoggingConfig(**raw.get("logging",
|
||||
{})),
|
||||
)
|
||||
|
|
@ -0,0 +1,171 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
constants.py
|
||||
"""
|
||||
|
||||
|
||||
from typing import Literal
|
||||
|
||||
|
||||
Severity = Literal["critical", "high", "medium", "low"]
|
||||
OutputFormat = Literal["console", "json", "sarif", "csv"]
|
||||
RedactionStyle = Literal["partial", "full", "none"]
|
||||
ScanTargetType = Literal["file", "database", "network"]
|
||||
|
||||
SEVERITY_ORDER: dict[Severity,
|
||||
int] = {
|
||||
"critical": 0,
|
||||
"high": 1,
|
||||
"medium": 2,
|
||||
"low": 3,
|
||||
}
|
||||
|
||||
SEVERITY_SCORE_THRESHOLDS: list[tuple[float,
|
||||
Severity]] = [
|
||||
(0.85,
|
||||
"critical"),
|
||||
(0.65,
|
||||
"high"),
|
||||
(0.40,
|
||||
"medium"),
|
||||
(0.20,
|
||||
"low"),
|
||||
]
|
||||
|
||||
COMPLIANCE_FRAMEWORKS: list[str] = [
|
||||
"HIPAA",
|
||||
"PCI_DSS",
|
||||
"GDPR",
|
||||
"CCPA",
|
||||
"SOX",
|
||||
"GLBA",
|
||||
"FERPA",
|
||||
]
|
||||
|
||||
DEFAULT_CONTEXT_WINDOW_TOKENS: int = 10
|
||||
DEFAULT_MIN_CONFIDENCE: float = 0.20
|
||||
DEFAULT_ENTROPY_THRESHOLD: float = 7.2
|
||||
DEFAULT_DNS_ENTROPY_THRESHOLD: float = 4.0
|
||||
DEFAULT_MAX_FILE_SIZE_MB: int = 100
|
||||
DEFAULT_DB_SAMPLE_PERCENTAGE: int = 5
|
||||
DEFAULT_DB_MAX_ROWS: int = 10000
|
||||
DEFAULT_DB_TIMEOUT_SECONDS: int = 30
|
||||
|
||||
CHECKSUM_BOOST: float = 0.30
|
||||
CONTEXT_BOOST_MAX: float = 0.35
|
||||
CONTEXT_BOOST_MIN_FLOOR: float = 0.40
|
||||
COOCCURRENCE_BOOST: float = 0.15
|
||||
|
||||
KNOWN_TEST_VALUES: frozenset[str] = frozenset(
|
||||
{
|
||||
"123-45-6789",
|
||||
"000-00-0000",
|
||||
"078-05-1120",
|
||||
"219-09-9999",
|
||||
"4111111111111111",
|
||||
"5500000000000004",
|
||||
"340000000000009",
|
||||
"6011000000000004",
|
||||
"test@example.com",
|
||||
"user@test.com",
|
||||
}
|
||||
)
|
||||
|
||||
DEFAULT_EXCLUDE_PATTERNS: list[str] = [
|
||||
"*.pyc",
|
||||
"__pycache__",
|
||||
".git",
|
||||
"node_modules",
|
||||
".venv",
|
||||
".env",
|
||||
"*.egg-info",
|
||||
]
|
||||
|
||||
SCANNABLE_EXTENSIONS: frozenset[str] = frozenset(
|
||||
{
|
||||
".pdf",
|
||||
".docx",
|
||||
".xlsx",
|
||||
".xls",
|
||||
".csv",
|
||||
".json",
|
||||
".xml",
|
||||
".yaml",
|
||||
".yml",
|
||||
".txt",
|
||||
".log",
|
||||
".cfg",
|
||||
".ini",
|
||||
".toml",
|
||||
".conf",
|
||||
".eml",
|
||||
".msg",
|
||||
".parquet",
|
||||
".avro",
|
||||
".md",
|
||||
".rst",
|
||||
".html",
|
||||
".htm",
|
||||
".tsv",
|
||||
".py",
|
||||
".js",
|
||||
".ts",
|
||||
".go",
|
||||
".rb",
|
||||
".java",
|
||||
".c",
|
||||
".cpp",
|
||||
".h",
|
||||
".hpp",
|
||||
".rs",
|
||||
".env",
|
||||
".sh",
|
||||
".bat",
|
||||
".ps1",
|
||||
".tf",
|
||||
".hcl",
|
||||
}
|
||||
)
|
||||
|
||||
TEXT_DB_COLUMN_TYPES_PG: frozenset[str] = frozenset(
|
||||
{
|
||||
"text",
|
||||
"character varying",
|
||||
"character",
|
||||
"json",
|
||||
"jsonb",
|
||||
"varchar",
|
||||
}
|
||||
)
|
||||
|
||||
TEXT_DB_COLUMN_TYPES_MYSQL: frozenset[str] = frozenset(
|
||||
{
|
||||
"varchar",
|
||||
"text",
|
||||
"mediumtext",
|
||||
"longtext",
|
||||
"json",
|
||||
"char",
|
||||
"tinytext",
|
||||
}
|
||||
)
|
||||
|
||||
SEVERITY_COLORS: dict[Severity,
|
||||
str] = {
|
||||
"critical": "bold red",
|
||||
"high": "red",
|
||||
"medium": "yellow",
|
||||
"low": "green",
|
||||
}
|
||||
|
||||
SARIF_SEVERITY_MAP: dict[Severity,
|
||||
str] = {
|
||||
"critical": "error",
|
||||
"high": "error",
|
||||
"medium": "warning",
|
||||
"low": "note",
|
||||
}
|
||||
|
||||
MAX_ARCHIVE_DEPTH: int = 3
|
||||
MAX_ARCHIVE_MEMBER_SIZE_MB: int = 50
|
||||
ZIP_BOMB_RATIO_THRESHOLD: int = 100
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
__init__.py
|
||||
"""
|
||||
|
|
@ -0,0 +1,51 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
base.py
|
||||
"""
|
||||
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Protocol
|
||||
from collections.abc import Callable
|
||||
|
||||
|
||||
@dataclass(frozen = True, slots = True)
|
||||
class DetectionRule:
|
||||
"""
|
||||
A single detection rule combining regex, validation, and context
|
||||
"""
|
||||
rule_id: str
|
||||
rule_name: str
|
||||
pattern: re.Pattern[str]
|
||||
base_score: float
|
||||
context_keywords: list[str] = field(default_factory = list)
|
||||
validator: Callable[[str], bool] | None = None
|
||||
compliance_frameworks: list[str] = field(default_factory = list)
|
||||
severity_override: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen = True, slots = True)
|
||||
class DetectorMatch:
|
||||
"""
|
||||
A raw match from a detector before scoring
|
||||
"""
|
||||
rule_id: str
|
||||
rule_name: str
|
||||
start: int
|
||||
end: int
|
||||
matched_text: str
|
||||
score: float
|
||||
context_keywords: list[str] = field(default_factory = list)
|
||||
compliance_frameworks: list[str] = field(default_factory = list)
|
||||
|
||||
|
||||
class Detector(Protocol):
|
||||
"""
|
||||
Protocol for all detection strategies
|
||||
"""
|
||||
def detect(self, text: str) -> list[DetectorMatch]:
|
||||
"""
|
||||
Scan text and return all matches
|
||||
"""
|
||||
...
|
||||
|
|
@ -0,0 +1,142 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
context.py
|
||||
"""
|
||||
|
||||
|
||||
from dlp_scanner.constants import (
|
||||
CONTEXT_BOOST_MAX,
|
||||
CONTEXT_BOOST_MIN_FLOOR,
|
||||
COOCCURRENCE_BOOST,
|
||||
DEFAULT_CONTEXT_WINDOW_TOKENS,
|
||||
)
|
||||
from dlp_scanner.detectors.base import DetectorMatch
|
||||
|
||||
|
||||
def apply_context_boost(
|
||||
text: str,
|
||||
matches: list[DetectorMatch],
|
||||
window_tokens: int = DEFAULT_CONTEXT_WINDOW_TOKENS,
|
||||
) -> list[DetectorMatch]:
|
||||
"""
|
||||
Boost match scores based on nearby context keywords
|
||||
"""
|
||||
if not matches:
|
||||
return matches
|
||||
|
||||
tokens = text.lower().split()
|
||||
boosted: list[DetectorMatch] = []
|
||||
|
||||
for match in matches:
|
||||
if not match.context_keywords:
|
||||
boosted.append(match)
|
||||
continue
|
||||
|
||||
char_to_token = _char_offset_to_token_index(text, match.start)
|
||||
window_start = max(0, char_to_token - window_tokens)
|
||||
window_end = min(len(tokens), char_to_token + window_tokens)
|
||||
window_text = " ".join(tokens[window_start : window_end])
|
||||
|
||||
boost = _compute_keyword_boost(
|
||||
window_text,
|
||||
match.context_keywords,
|
||||
window_tokens,
|
||||
)
|
||||
|
||||
new_score = min(1.0, match.score + boost)
|
||||
if boost > 0 and new_score < CONTEXT_BOOST_MIN_FLOOR:
|
||||
new_score = CONTEXT_BOOST_MIN_FLOOR
|
||||
|
||||
boosted.append(
|
||||
DetectorMatch(
|
||||
rule_id = match.rule_id,
|
||||
rule_name = match.rule_name,
|
||||
start = match.start,
|
||||
end = match.end,
|
||||
matched_text = match.matched_text,
|
||||
score = new_score,
|
||||
context_keywords = match.context_keywords,
|
||||
compliance_frameworks = match.compliance_frameworks,
|
||||
)
|
||||
)
|
||||
|
||||
return _apply_cooccurrence_boost(boosted)
|
||||
|
||||
|
||||
def _compute_keyword_boost(
|
||||
window_text: str,
|
||||
keywords: list[str],
|
||||
window_tokens: int,
|
||||
) -> float:
|
||||
"""
|
||||
Compute score boost based on keyword proximity
|
||||
"""
|
||||
best_boost = 0.0
|
||||
|
||||
for keyword in keywords:
|
||||
kw_lower = keyword.lower()
|
||||
pos = window_text.find(kw_lower)
|
||||
if pos < 0:
|
||||
continue
|
||||
|
||||
center = len(window_text) // 2
|
||||
distance = abs(pos - center)
|
||||
max_distance = window_tokens * 5
|
||||
|
||||
proximity_factor = 1.0 - min(1.0, distance / max(1, max_distance))
|
||||
boost = CONTEXT_BOOST_MAX * proximity_factor
|
||||
best_boost = max(best_boost, boost)
|
||||
|
||||
return best_boost
|
||||
|
||||
|
||||
def _apply_cooccurrence_boost(
|
||||
matches: list[DetectorMatch],
|
||||
) -> list[DetectorMatch]:
|
||||
"""
|
||||
Boost scores when multiple PII types appear near each other
|
||||
"""
|
||||
if len(matches) < 2:
|
||||
return matches
|
||||
|
||||
proximity_threshold = 500
|
||||
boosted: list[DetectorMatch] = []
|
||||
|
||||
for i, match in enumerate(matches):
|
||||
has_neighbor = False
|
||||
for j, other in enumerate(matches):
|
||||
if i == j:
|
||||
continue
|
||||
if other.rule_id == match.rule_id:
|
||||
continue
|
||||
distance = abs(match.start - other.start)
|
||||
if distance < proximity_threshold:
|
||||
has_neighbor = True
|
||||
break
|
||||
|
||||
if has_neighbor:
|
||||
new_score = min(1.0, match.score + COOCCURRENCE_BOOST)
|
||||
boosted.append(
|
||||
DetectorMatch(
|
||||
rule_id = match.rule_id,
|
||||
rule_name = match.rule_name,
|
||||
start = match.start,
|
||||
end = match.end,
|
||||
matched_text = match.matched_text,
|
||||
score = new_score,
|
||||
context_keywords = match.context_keywords,
|
||||
compliance_frameworks = match.compliance_frameworks,
|
||||
)
|
||||
)
|
||||
else:
|
||||
boosted.append(match)
|
||||
|
||||
return boosted
|
||||
|
||||
|
||||
def _char_offset_to_token_index(text: str, char_offset: int) -> int:
|
||||
"""
|
||||
Convert a character offset to an approximate token index
|
||||
"""
|
||||
prefix = text[: char_offset]
|
||||
return len(prefix.split())
|
||||
|
|
@ -0,0 +1,126 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
entropy.py
|
||||
"""
|
||||
|
||||
|
||||
import math
|
||||
from collections import Counter
|
||||
|
||||
from dlp_scanner.constants import DEFAULT_ENTROPY_THRESHOLD
|
||||
from dlp_scanner.detectors.base import DetectorMatch
|
||||
|
||||
|
||||
WINDOW_SIZE: int = 256
|
||||
WINDOW_STEP: int = 128
|
||||
|
||||
|
||||
def shannon_entropy(data: bytes) -> float:
|
||||
"""
|
||||
Calculate Shannon entropy in bits per byte
|
||||
"""
|
||||
if not data:
|
||||
return 0.0
|
||||
|
||||
counts = Counter(data)
|
||||
total = len(data)
|
||||
return -sum(
|
||||
(c / total) * math.log2(c / total) for c in counts.values()
|
||||
)
|
||||
|
||||
|
||||
def shannon_entropy_str(text: str) -> float:
|
||||
"""
|
||||
Calculate Shannon entropy for a string
|
||||
"""
|
||||
return shannon_entropy(text.encode("utf-8"))
|
||||
|
||||
|
||||
def detect_high_entropy_regions(
|
||||
data: bytes,
|
||||
threshold: float = DEFAULT_ENTROPY_THRESHOLD,
|
||||
window_size: int = WINDOW_SIZE,
|
||||
step: int = WINDOW_STEP,
|
||||
) -> list[tuple[int,
|
||||
int,
|
||||
float]]:
|
||||
"""
|
||||
Find regions of high entropy using a sliding window
|
||||
|
||||
Returns list of (start_offset, end_offset, entropy_value)
|
||||
"""
|
||||
if len(data) < window_size:
|
||||
h = shannon_entropy(data)
|
||||
if h >= threshold:
|
||||
return [(0, len(data), h)]
|
||||
return []
|
||||
|
||||
regions: list[tuple[int, int, float]] = []
|
||||
i = 0
|
||||
|
||||
while i + window_size <= len(data):
|
||||
window = data[i : i + window_size]
|
||||
h = shannon_entropy(window)
|
||||
|
||||
if h >= threshold:
|
||||
end = i + window_size
|
||||
while end + step <= len(data):
|
||||
next_window = data[end - window_size + step : end + step]
|
||||
next_h = shannon_entropy(next_window)
|
||||
if next_h < threshold:
|
||||
break
|
||||
h = max(h, next_h)
|
||||
end += step
|
||||
|
||||
regions.append((i, end, h))
|
||||
i = end
|
||||
else:
|
||||
i += step
|
||||
|
||||
return regions
|
||||
|
||||
|
||||
class EntropyDetector:
|
||||
"""
|
||||
Detects high-entropy data that may indicate encrypted
|
||||
or compressed content
|
||||
"""
|
||||
def __init__(
|
||||
self,
|
||||
threshold: float = DEFAULT_ENTROPY_THRESHOLD,
|
||||
) -> None:
|
||||
self._threshold = threshold
|
||||
|
||||
def detect(self, text: str) -> list[DetectorMatch]:
|
||||
"""
|
||||
Scan text for high-entropy regions
|
||||
"""
|
||||
data = text.encode("utf-8")
|
||||
regions = detect_high_entropy_regions(
|
||||
data,
|
||||
threshold = self._threshold,
|
||||
)
|
||||
|
||||
matches: list[DetectorMatch] = []
|
||||
for start, end, entropy_val in regions:
|
||||
score = min(
|
||||
1.0,
|
||||
(entropy_val - self._threshold) /
|
||||
(8.0 - self._threshold) * 0.5 + 0.5,
|
||||
)
|
||||
|
||||
matches.append(
|
||||
DetectorMatch(
|
||||
rule_id = "NET_HIGH_ENTROPY",
|
||||
rule_name = "High Entropy Data",
|
||||
start = start,
|
||||
end = end,
|
||||
matched_text =
|
||||
f"[{end - start} bytes, H={entropy_val:.2f}]",
|
||||
score = score,
|
||||
context_keywords = [],
|
||||
compliance_frameworks = [],
|
||||
)
|
||||
)
|
||||
|
||||
return matches
|
||||
|
|
@ -0,0 +1,68 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
pattern.py
|
||||
"""
|
||||
|
||||
|
||||
from dlp_scanner.constants import CHECKSUM_BOOST, KNOWN_TEST_VALUES
|
||||
from dlp_scanner.detectors.base import (
|
||||
DetectionRule,
|
||||
DetectorMatch,
|
||||
)
|
||||
|
||||
|
||||
class PatternDetector:
|
||||
"""
|
||||
Detects sensitive data using regex patterns with optional
|
||||
checksum validation
|
||||
"""
|
||||
def __init__(
|
||||
self,
|
||||
rules: list[DetectionRule],
|
||||
allowlist_values: frozenset[str] | None = None,
|
||||
) -> None:
|
||||
self._rules = rules
|
||||
self._allowlist = allowlist_values or KNOWN_TEST_VALUES
|
||||
|
||||
def detect(self, text: str) -> list[DetectorMatch]:
|
||||
"""
|
||||
Scan text against all registered patterns
|
||||
"""
|
||||
matches: list[DetectorMatch] = []
|
||||
|
||||
for rule in self._rules:
|
||||
for m in rule.pattern.finditer(text):
|
||||
matched_text = m.group()
|
||||
|
||||
if self._is_allowlisted(matched_text):
|
||||
continue
|
||||
|
||||
score = rule.base_score
|
||||
|
||||
if rule.validator is not None:
|
||||
if rule.validator(matched_text):
|
||||
score = min(1.0, score + CHECKSUM_BOOST)
|
||||
else:
|
||||
continue
|
||||
|
||||
matches.append(
|
||||
DetectorMatch(
|
||||
rule_id = rule.rule_id,
|
||||
rule_name = rule.rule_name,
|
||||
start = m.start(),
|
||||
end = m.end(),
|
||||
matched_text = matched_text,
|
||||
score = score,
|
||||
context_keywords = rule.context_keywords,
|
||||
compliance_frameworks = rule.compliance_frameworks,
|
||||
)
|
||||
)
|
||||
|
||||
return matches
|
||||
|
||||
def _is_allowlisted(self, value: str) -> bool:
|
||||
"""
|
||||
Check if a matched value is in the allowlist
|
||||
"""
|
||||
normalized = value.strip()
|
||||
return normalized in self._allowlist
|
||||
|
|
@ -0,0 +1,112 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
registry.py
|
||||
"""
|
||||
|
||||
|
||||
import fnmatch
|
||||
|
||||
from dlp_scanner.detectors.base import (
|
||||
DetectionRule,
|
||||
DetectorMatch,
|
||||
)
|
||||
from dlp_scanner.detectors.context import (
|
||||
apply_context_boost,
|
||||
)
|
||||
from dlp_scanner.detectors.entropy import EntropyDetector
|
||||
from dlp_scanner.detectors.pattern import PatternDetector
|
||||
from dlp_scanner.detectors.rules.credentials import (
|
||||
CREDENTIAL_RULES,
|
||||
)
|
||||
from dlp_scanner.detectors.rules.financial import (
|
||||
FINANCIAL_RULES,
|
||||
)
|
||||
from dlp_scanner.detectors.rules.health import HEALTH_RULES
|
||||
from dlp_scanner.detectors.rules.pii import PII_RULES
|
||||
|
||||
|
||||
ALL_RULES: list[DetectionRule] = [
|
||||
*PII_RULES,
|
||||
*FINANCIAL_RULES,
|
||||
*CREDENTIAL_RULES,
|
||||
*HEALTH_RULES,
|
||||
]
|
||||
|
||||
|
||||
class DetectorRegistry:
|
||||
"""
|
||||
Central registry that loads, filters, and runs all detectors
|
||||
"""
|
||||
def __init__(
|
||||
self,
|
||||
enable_patterns: list[str] | None = None,
|
||||
disable_patterns: list[str] | None = None,
|
||||
allowlist_values: frozenset[str] | None = None,
|
||||
context_window_tokens: int = 10,
|
||||
entropy_threshold: float = 7.2,
|
||||
enable_entropy: bool = True,
|
||||
) -> None:
|
||||
active_rules = _filter_rules(
|
||||
ALL_RULES,
|
||||
enable_patterns or ["*"],
|
||||
disable_patterns or [],
|
||||
)
|
||||
|
||||
self._pattern_detector = PatternDetector(
|
||||
rules = active_rules,
|
||||
allowlist_values = allowlist_values,
|
||||
)
|
||||
self._entropy_detector = (
|
||||
EntropyDetector(threshold = entropy_threshold)
|
||||
if enable_entropy else None
|
||||
)
|
||||
self._context_window = context_window_tokens
|
||||
|
||||
def detect(self, text: str) -> list[DetectorMatch]:
|
||||
"""
|
||||
Run all detectors against text and return scored matches
|
||||
"""
|
||||
matches = self._pattern_detector.detect(text)
|
||||
matches = apply_context_boost(
|
||||
text,
|
||||
matches,
|
||||
window_tokens = self._context_window,
|
||||
)
|
||||
|
||||
if self._entropy_detector is not None:
|
||||
entropy_matches = (self._entropy_detector.detect(text))
|
||||
matches.extend(entropy_matches)
|
||||
|
||||
return matches
|
||||
|
||||
@property
|
||||
def rule_count(self) -> int:
|
||||
"""
|
||||
Return the number of active pattern rules
|
||||
"""
|
||||
return len(self._pattern_detector._rules)
|
||||
|
||||
|
||||
def _filter_rules(
|
||||
rules: list[DetectionRule],
|
||||
enable_patterns: list[str],
|
||||
disable_patterns: list[str],
|
||||
) -> list[DetectionRule]:
|
||||
"""
|
||||
Filter rules by enable/disable glob patterns
|
||||
"""
|
||||
filtered: list[DetectionRule] = []
|
||||
|
||||
for rule in rules:
|
||||
enabled = any(
|
||||
fnmatch.fnmatch(rule.rule_id,
|
||||
pat) for pat in enable_patterns
|
||||
)
|
||||
disabled = any(
|
||||
fnmatch.fnmatch(rule.rule_id,
|
||||
pat) for pat in disable_patterns
|
||||
)
|
||||
if enabled and not disabled:
|
||||
filtered.append(rule)
|
||||
|
||||
return filtered
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
__init__.py
|
||||
"""
|
||||
|
|
@ -0,0 +1,157 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
credentials.py
|
||||
"""
|
||||
|
||||
|
||||
import re
|
||||
|
||||
from dlp_scanner.detectors.base import DetectionRule
|
||||
|
||||
|
||||
AWS_ACCESS_KEY_PATTERN = re.compile(r"\b((?:AKIA|ASIA)[0-9A-Z]{16})\b")
|
||||
|
||||
GITHUB_CLASSIC_PAT_PATTERN = re.compile(r"\bghp_[a-zA-Z0-9]{36}\b")
|
||||
|
||||
GITHUB_FINE_GRAINED_PATTERN = re.compile(
|
||||
r"\bgithub_pat_[a-zA-Z0-9]{22}_[a-zA-Z0-9]{59}\b"
|
||||
)
|
||||
|
||||
GITHUB_OAUTH_PATTERN = re.compile(r"\bgho_[a-zA-Z0-9]{36}\b")
|
||||
|
||||
GITHUB_APP_PATTERN = re.compile(r"\bghs_[a-zA-Z0-9]{36}\b")
|
||||
|
||||
JWT_PATTERN = re.compile(
|
||||
r"\beyJ[a-zA-Z0-9_-]+\.eyJ[a-zA-Z0-9_-]+\.[a-zA-Z0-9_-]+\b"
|
||||
)
|
||||
|
||||
STRIPE_KEY_PATTERN = re.compile(
|
||||
r"\b(?:sk|pk)_(?:test|live)_[a-zA-Z0-9]{24,}\b"
|
||||
)
|
||||
|
||||
SLACK_TOKEN_PATTERN = re.compile(r"\bxox[baprs]-[a-zA-Z0-9\-]{10,48}\b")
|
||||
|
||||
GENERIC_API_KEY_PATTERN = re.compile(
|
||||
r"(?i)(?:api[_\-]?key|apikey|api[_\-]?token|access[_\-]?key|secret[_\-]?key)"
|
||||
r"\s*[:=]\s*['\"]?"
|
||||
r"([a-zA-Z0-9\-_.]{20,64})"
|
||||
r"['\"]?"
|
||||
)
|
||||
|
||||
PRIVATE_KEY_PATTERN = re.compile(
|
||||
r"-----BEGIN (?:RSA |EC |DSA |OPENSSH )?PRIVATE KEY-----"
|
||||
)
|
||||
|
||||
API_KEY_CONTEXT = [
|
||||
"api_key",
|
||||
"apikey",
|
||||
"api key",
|
||||
"secret",
|
||||
"token",
|
||||
"authorization",
|
||||
"bearer",
|
||||
"credential",
|
||||
"password",
|
||||
"access_key",
|
||||
]
|
||||
|
||||
CREDENTIAL_RULES: list[DetectionRule] = [
|
||||
DetectionRule(
|
||||
rule_id = "CRED_AWS_ACCESS_KEY",
|
||||
rule_name = "AWS Access Key ID",
|
||||
pattern = AWS_ACCESS_KEY_PATTERN,
|
||||
base_score = 0.85,
|
||||
context_keywords = [
|
||||
"aws",
|
||||
"amazon",
|
||||
"access_key",
|
||||
"aws_access_key_id",
|
||||
],
|
||||
),
|
||||
DetectionRule(
|
||||
rule_id = "CRED_GITHUB_TOKEN",
|
||||
rule_name = "GitHub Personal Access Token",
|
||||
pattern = GITHUB_CLASSIC_PAT_PATTERN,
|
||||
base_score = 0.90,
|
||||
context_keywords = ["github",
|
||||
"token",
|
||||
"pat"],
|
||||
),
|
||||
DetectionRule(
|
||||
rule_id = "CRED_GITHUB_FINE_GRAINED",
|
||||
rule_name = "GitHub Fine-Grained PAT",
|
||||
pattern = GITHUB_FINE_GRAINED_PATTERN,
|
||||
base_score = 0.90,
|
||||
context_keywords = ["github",
|
||||
"token"],
|
||||
),
|
||||
DetectionRule(
|
||||
rule_id = "CRED_GITHUB_OAUTH",
|
||||
rule_name = "GitHub OAuth Token",
|
||||
pattern = GITHUB_OAUTH_PATTERN,
|
||||
base_score = 0.90,
|
||||
context_keywords = ["github",
|
||||
"oauth"],
|
||||
),
|
||||
DetectionRule(
|
||||
rule_id = "CRED_GITHUB_APP",
|
||||
rule_name = "GitHub App Token",
|
||||
pattern = GITHUB_APP_PATTERN,
|
||||
base_score = 0.90,
|
||||
context_keywords = ["github",
|
||||
"app"],
|
||||
),
|
||||
DetectionRule(
|
||||
rule_id = "CRED_JWT",
|
||||
rule_name = "JSON Web Token",
|
||||
pattern = JWT_PATTERN,
|
||||
base_score = 0.70,
|
||||
context_keywords = [
|
||||
"jwt",
|
||||
"token",
|
||||
"bearer",
|
||||
"authorization",
|
||||
],
|
||||
),
|
||||
DetectionRule(
|
||||
rule_id = "CRED_STRIPE_KEY",
|
||||
rule_name = "Stripe API Key",
|
||||
pattern = STRIPE_KEY_PATTERN,
|
||||
base_score = 0.90,
|
||||
context_keywords = [
|
||||
"stripe",
|
||||
"payment",
|
||||
"api_key",
|
||||
],
|
||||
),
|
||||
DetectionRule(
|
||||
rule_id = "CRED_SLACK_TOKEN",
|
||||
rule_name = "Slack Token",
|
||||
pattern = SLACK_TOKEN_PATTERN,
|
||||
base_score = 0.85,
|
||||
context_keywords = [
|
||||
"slack",
|
||||
"token",
|
||||
"webhook",
|
||||
],
|
||||
),
|
||||
DetectionRule(
|
||||
rule_id = "CRED_GENERIC_API_KEY",
|
||||
rule_name = "Generic API Key",
|
||||
pattern = GENERIC_API_KEY_PATTERN,
|
||||
base_score = 0.50,
|
||||
context_keywords = API_KEY_CONTEXT,
|
||||
),
|
||||
DetectionRule(
|
||||
rule_id = "CRED_PRIVATE_KEY",
|
||||
rule_name = "Private Key",
|
||||
pattern = PRIVATE_KEY_PATTERN,
|
||||
base_score = 0.95,
|
||||
context_keywords = [
|
||||
"private key",
|
||||
"rsa",
|
||||
"ssh",
|
||||
"certificate",
|
||||
],
|
||||
),
|
||||
]
|
||||
|
|
@ -0,0 +1,184 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
financial.py
|
||||
"""
|
||||
|
||||
|
||||
import re
|
||||
|
||||
from dlp_scanner.detectors.base import DetectionRule
|
||||
|
||||
|
||||
VISA_PATTERN = re.compile(
|
||||
r"\b4[0-9]{3}[-\s]?[0-9]{4}[-\s]?[0-9]{4}[-\s]?[0-9]{4}\b"
|
||||
)
|
||||
|
||||
MASTERCARD_PATTERN = re.compile(
|
||||
r"\b(?:5[1-5][0-9]{2}|222[1-9]|22[3-9][0-9]|2[3-6][0-9]{2}|27[01][0-9]|2720)"
|
||||
r"[-\s]?[0-9]{4}[-\s]?[0-9]{4}[-\s]?[0-9]{4}\b"
|
||||
)
|
||||
|
||||
AMEX_PATTERN = re.compile(r"\b3[47][0-9]{2}[-\s]?[0-9]{6}[-\s]?[0-9]{5}\b")
|
||||
|
||||
DISCOVER_PATTERN = re.compile(
|
||||
r"\b6(?:011|5[0-9]{2})[-\s]?[0-9]{4}[-\s]?[0-9]{4}[-\s]?[0-9]{4}\b"
|
||||
)
|
||||
|
||||
IBAN_PATTERN = re.compile(
|
||||
r"\b[A-Z]{2}\d{2}[A-Z0-9]{4}\d{7}[A-Z0-9]{0,16}\b"
|
||||
)
|
||||
|
||||
NHS_PATTERN = re.compile(r"\b\d{3}[-\s]?\d{3}[-\s]?\d{4}\b")
|
||||
|
||||
|
||||
def luhn_check(number: str) -> bool:
|
||||
"""
|
||||
Validate a number using the Luhn algorithm
|
||||
"""
|
||||
digits = [int(d) for d in number if d.isdigit()]
|
||||
if len(digits) < 13:
|
||||
return False
|
||||
|
||||
odd_digits = digits[-1 ::-2]
|
||||
even_digits = digits[-2 ::-2]
|
||||
total = sum(odd_digits)
|
||||
for d in even_digits:
|
||||
total += sum(divmod(d * 2, 10))
|
||||
return total % 10 == 0
|
||||
|
||||
|
||||
def iban_check(value: str) -> bool:
|
||||
"""
|
||||
Validate an IBAN using the mod-97 algorithm
|
||||
"""
|
||||
cleaned = value.replace(" ", "").upper()
|
||||
if len(cleaned) < 15 or len(cleaned) > 34:
|
||||
return False
|
||||
|
||||
rearranged = cleaned[4 :] + cleaned[: 4]
|
||||
numeric = ""
|
||||
for char in rearranged:
|
||||
if char.isalpha():
|
||||
numeric += str(ord(char) - ord("A") + 10)
|
||||
else:
|
||||
numeric += char
|
||||
|
||||
return int(numeric) % 97 == 1
|
||||
|
||||
|
||||
def nhs_check(value: str) -> bool:
|
||||
"""
|
||||
Validate a UK NHS number using mod-11
|
||||
"""
|
||||
digits = value.replace("-", "").replace(" ", "")
|
||||
if len(digits) != 10 or not digits.isdigit():
|
||||
return False
|
||||
|
||||
weights = range(10, 1, -1)
|
||||
total = sum(
|
||||
int(d) * w for d, w in zip(digits[: 9], weights, strict = False)
|
||||
)
|
||||
remainder = 11 - (total % 11)
|
||||
if remainder == 11:
|
||||
remainder = 0
|
||||
if remainder == 10:
|
||||
return False
|
||||
return remainder == int(digits[9])
|
||||
|
||||
|
||||
CREDIT_CARD_CONTEXT = [
|
||||
"credit card",
|
||||
"card number",
|
||||
"cc",
|
||||
"cvv",
|
||||
"cvc",
|
||||
"expiry",
|
||||
"expiration",
|
||||
"visa",
|
||||
"mastercard",
|
||||
"amex",
|
||||
"card no",
|
||||
"payment card",
|
||||
"pan",
|
||||
]
|
||||
|
||||
IBAN_CONTEXT = [
|
||||
"iban",
|
||||
"bank account",
|
||||
"account number",
|
||||
"swift",
|
||||
"bic",
|
||||
"wire transfer",
|
||||
"bank transfer",
|
||||
]
|
||||
|
||||
NHS_CONTEXT = [
|
||||
"nhs",
|
||||
"nhs number",
|
||||
"national health",
|
||||
"health service",
|
||||
"patient id",
|
||||
"patient number",
|
||||
]
|
||||
|
||||
FINANCIAL_RULES: list[DetectionRule] = [
|
||||
DetectionRule(
|
||||
rule_id = "FIN_CREDIT_CARD_VISA",
|
||||
rule_name = "Visa Credit Card Number",
|
||||
pattern = VISA_PATTERN,
|
||||
base_score = 0.50,
|
||||
context_keywords = CREDIT_CARD_CONTEXT,
|
||||
validator = luhn_check,
|
||||
compliance_frameworks = ["PCI_DSS",
|
||||
"GLBA"],
|
||||
),
|
||||
DetectionRule(
|
||||
rule_id = "FIN_CREDIT_CARD_MC",
|
||||
rule_name = "Mastercard Credit Card Number",
|
||||
pattern = MASTERCARD_PATTERN,
|
||||
base_score = 0.50,
|
||||
context_keywords = CREDIT_CARD_CONTEXT,
|
||||
validator = luhn_check,
|
||||
compliance_frameworks = ["PCI_DSS",
|
||||
"GLBA"],
|
||||
),
|
||||
DetectionRule(
|
||||
rule_id = "FIN_CREDIT_CARD_AMEX",
|
||||
rule_name = "American Express Card Number",
|
||||
pattern = AMEX_PATTERN,
|
||||
base_score = 0.50,
|
||||
context_keywords = CREDIT_CARD_CONTEXT,
|
||||
validator = luhn_check,
|
||||
compliance_frameworks = ["PCI_DSS",
|
||||
"GLBA"],
|
||||
),
|
||||
DetectionRule(
|
||||
rule_id = "FIN_CREDIT_CARD_DISC",
|
||||
rule_name = "Discover Card Number",
|
||||
pattern = DISCOVER_PATTERN,
|
||||
base_score = 0.50,
|
||||
context_keywords = CREDIT_CARD_CONTEXT,
|
||||
validator = luhn_check,
|
||||
compliance_frameworks = ["PCI_DSS",
|
||||
"GLBA"],
|
||||
),
|
||||
DetectionRule(
|
||||
rule_id = "FIN_IBAN",
|
||||
rule_name = "IBAN Number",
|
||||
pattern = IBAN_PATTERN,
|
||||
base_score = 0.40,
|
||||
context_keywords = IBAN_CONTEXT,
|
||||
validator = iban_check,
|
||||
compliance_frameworks = ["GDPR",
|
||||
"GLBA"],
|
||||
),
|
||||
DetectionRule(
|
||||
rule_id = "FIN_NHS_NUMBER",
|
||||
rule_name = "UK NHS Number",
|
||||
pattern = NHS_PATTERN,
|
||||
base_score = 0.15,
|
||||
context_keywords = NHS_CONTEXT,
|
||||
validator = nhs_check,
|
||||
compliance_frameworks = ["GDPR"],
|
||||
),
|
||||
]
|
||||
|
|
@ -0,0 +1,140 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
health.py
|
||||
"""
|
||||
|
||||
|
||||
import re
|
||||
|
||||
from dlp_scanner.detectors.base import DetectionRule
|
||||
|
||||
|
||||
MEDICAL_RECORD_PATTERN = re.compile(
|
||||
r"\b(?:MRN|MR#|MED)\s*[-:#]?\s*\d{6,10}\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
DEA_NUMBER_PATTERN = re.compile(r"\b[A-Z][A-Z9]\d{7}\b")
|
||||
|
||||
NPI_PATTERN = re.compile(r"\b\d{10}\b")
|
||||
|
||||
PHI_CONTEXT_KEYWORDS = [
|
||||
"patient",
|
||||
"diagnosis",
|
||||
"treatment",
|
||||
"medical",
|
||||
"health",
|
||||
"hospital",
|
||||
"clinical",
|
||||
"physician",
|
||||
"prescription",
|
||||
"medication",
|
||||
"lab result",
|
||||
"blood type",
|
||||
"allergies",
|
||||
"insurance",
|
||||
"claim",
|
||||
"icd",
|
||||
"cpt",
|
||||
"hcpcs",
|
||||
"hipaa",
|
||||
"phi",
|
||||
"protected health",
|
||||
"discharge",
|
||||
"admission",
|
||||
"prognosis",
|
||||
]
|
||||
|
||||
MEDICAL_RECORD_CONTEXT = [
|
||||
"medical record",
|
||||
"mrn",
|
||||
"patient id",
|
||||
"chart number",
|
||||
"record number",
|
||||
"health record",
|
||||
"ehr",
|
||||
"emr",
|
||||
]
|
||||
|
||||
DEA_CONTEXT = [
|
||||
"dea",
|
||||
"dea number",
|
||||
"drug enforcement",
|
||||
"prescriber",
|
||||
"controlled substance",
|
||||
]
|
||||
|
||||
NPI_CONTEXT = [
|
||||
"npi",
|
||||
"national provider",
|
||||
"provider id",
|
||||
"provider number",
|
||||
"provider identifier",
|
||||
"cms",
|
||||
]
|
||||
|
||||
|
||||
def _validate_npi(value: str) -> bool:
|
||||
"""
|
||||
Validate an NPI using Luhn with the 80840 prefix
|
||||
"""
|
||||
digits = value.replace("-", "").replace(" ", "")
|
||||
if len(digits) != 10 or not digits.isdigit():
|
||||
return False
|
||||
|
||||
prefixed = "80840" + digits
|
||||
total = 0
|
||||
for i, d in enumerate(reversed(prefixed)):
|
||||
n = int(d)
|
||||
if i % 2 == 1:
|
||||
n *= 2
|
||||
if n > 9:
|
||||
n -= 9
|
||||
total += n
|
||||
return total % 10 == 0
|
||||
|
||||
|
||||
def _validate_dea_number(value: str) -> bool:
|
||||
"""
|
||||
Validate a DEA number using its check digit algorithm
|
||||
"""
|
||||
if len(value) != 9:
|
||||
return False
|
||||
digits = value[2 :]
|
||||
if not digits.isdigit():
|
||||
return False
|
||||
|
||||
odd_sum = (int(digits[0]) + int(digits[2]) + int(digits[4]))
|
||||
even_sum = (int(digits[1]) + int(digits[3]) + int(digits[5]))
|
||||
check = (odd_sum + even_sum * 2) % 10
|
||||
return check == int(digits[6])
|
||||
|
||||
|
||||
HEALTH_RULES: list[DetectionRule] = [
|
||||
DetectionRule(
|
||||
rule_id = "HEALTH_MEDICAL_RECORD",
|
||||
rule_name = "Medical Record Number",
|
||||
pattern = MEDICAL_RECORD_PATTERN,
|
||||
base_score = 0.55,
|
||||
context_keywords = MEDICAL_RECORD_CONTEXT,
|
||||
compliance_frameworks = ["HIPAA"],
|
||||
),
|
||||
DetectionRule(
|
||||
rule_id = "HEALTH_DEA_NUMBER",
|
||||
rule_name = "DEA Registration Number",
|
||||
pattern = DEA_NUMBER_PATTERN,
|
||||
base_score = 0.35,
|
||||
context_keywords = DEA_CONTEXT,
|
||||
validator = _validate_dea_number,
|
||||
compliance_frameworks = ["HIPAA"],
|
||||
),
|
||||
DetectionRule(
|
||||
rule_id = "HEALTH_NPI",
|
||||
rule_name = "National Provider Identifier",
|
||||
pattern = NPI_PATTERN,
|
||||
base_score = 0.10,
|
||||
context_keywords = NPI_CONTEXT,
|
||||
validator = _validate_npi,
|
||||
compliance_frameworks = ["HIPAA"],
|
||||
),
|
||||
]
|
||||
|
|
@ -0,0 +1,209 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
pii.py
|
||||
"""
|
||||
|
||||
|
||||
import re
|
||||
|
||||
from dlp_scanner.detectors.base import DetectionRule
|
||||
|
||||
|
||||
SSN_PATTERN = re.compile(
|
||||
r"\b(?!000|666|9\d{2})\d{3}"
|
||||
r"[-\s]?"
|
||||
r"(?!00)\d{2}"
|
||||
r"[-\s]?"
|
||||
r"(?!0000)\d{4}\b"
|
||||
)
|
||||
|
||||
EMAIL_PATTERN = re.compile(
|
||||
r"\b[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}\b"
|
||||
)
|
||||
|
||||
PHONE_US_PATTERN = re.compile(
|
||||
r"\b(?:\+?1[-.\s]?)?"
|
||||
r"(?:\(?[2-9]\d{2}\)?[-.\s]?)"
|
||||
r"[2-9]\d{2}[-.\s]?\d{4}\b"
|
||||
)
|
||||
|
||||
PHONE_E164_PATTERN = re.compile(r"\+[1-9]\d{6,14}\b")
|
||||
|
||||
PASSPORT_US_PATTERN = re.compile(r"\b[A-Z]{1,2}\d{6,7}\b")
|
||||
|
||||
PASSPORT_UK_PATTERN = re.compile(r"\b\d{9}\b")
|
||||
|
||||
IPV4_PATTERN = re.compile(
|
||||
r"\b(?:(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}"
|
||||
r"(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\b"
|
||||
)
|
||||
|
||||
DRIVERS_LICENSE_CA_PATTERN = re.compile(r"\b[A-Z]\d{7}\b")
|
||||
DRIVERS_LICENSE_FL_PATTERN = re.compile(r"\b[A-Z]\d{12}\b")
|
||||
DRIVERS_LICENSE_IL_PATTERN = re.compile(r"\b[A-Z]\d{11}\b")
|
||||
|
||||
|
||||
def _validate_ssn(value: str) -> bool:
|
||||
"""
|
||||
Validate SSN area, group, and serial numbers
|
||||
"""
|
||||
digits = value.replace("-", "").replace(" ", "")
|
||||
if len(digits) != 9 or not digits.isdigit():
|
||||
return False
|
||||
|
||||
area = int(digits[0 : 3])
|
||||
group = int(digits[3 : 5])
|
||||
serial = int(digits[5 : 9])
|
||||
|
||||
if area in {0, 666} or area >= 900:
|
||||
return False
|
||||
if group == 0:
|
||||
return False
|
||||
return serial != 0
|
||||
|
||||
|
||||
SSN_CONTEXT = [
|
||||
"ssn",
|
||||
"social security",
|
||||
"social security number",
|
||||
"ss#",
|
||||
"taxpayer id",
|
||||
"sin",
|
||||
"tax id",
|
||||
]
|
||||
|
||||
EMAIL_CONTEXT = [
|
||||
"email",
|
||||
"e-mail",
|
||||
"mail",
|
||||
"contact",
|
||||
"reach at",
|
||||
]
|
||||
|
||||
PHONE_CONTEXT = [
|
||||
"phone",
|
||||
"mobile",
|
||||
"cell",
|
||||
"tel",
|
||||
"telephone",
|
||||
"fax",
|
||||
"contact number",
|
||||
"call",
|
||||
]
|
||||
|
||||
PASSPORT_CONTEXT = [
|
||||
"passport",
|
||||
"pass no",
|
||||
"travel document",
|
||||
"passport number",
|
||||
"document number",
|
||||
]
|
||||
|
||||
DRIVERS_LICENSE_CONTEXT = [
|
||||
"driver's license",
|
||||
"drivers license",
|
||||
"driver license",
|
||||
"dl#",
|
||||
"dl number",
|
||||
"license number",
|
||||
"licence number",
|
||||
]
|
||||
|
||||
PII_RULES: list[DetectionRule] = [
|
||||
DetectionRule(
|
||||
rule_id = "PII_SSN",
|
||||
rule_name = "US Social Security Number",
|
||||
pattern = SSN_PATTERN,
|
||||
base_score = 0.45,
|
||||
context_keywords = SSN_CONTEXT,
|
||||
validator = _validate_ssn,
|
||||
compliance_frameworks = [
|
||||
"HIPAA",
|
||||
"CCPA",
|
||||
"GLBA",
|
||||
"GDPR",
|
||||
],
|
||||
),
|
||||
DetectionRule(
|
||||
rule_id = "PII_EMAIL",
|
||||
rule_name = "Email Address",
|
||||
pattern = EMAIL_PATTERN,
|
||||
base_score = 0.30,
|
||||
context_keywords = EMAIL_CONTEXT,
|
||||
compliance_frameworks = ["GDPR",
|
||||
"CCPA"],
|
||||
),
|
||||
DetectionRule(
|
||||
rule_id = "PII_PHONE",
|
||||
rule_name = "US Phone Number",
|
||||
pattern = PHONE_US_PATTERN,
|
||||
base_score = 0.25,
|
||||
context_keywords = PHONE_CONTEXT,
|
||||
compliance_frameworks = [
|
||||
"GDPR",
|
||||
"CCPA",
|
||||
"HIPAA",
|
||||
],
|
||||
),
|
||||
DetectionRule(
|
||||
rule_id = "PII_PHONE_INTL",
|
||||
rule_name = "International Phone Number",
|
||||
pattern = PHONE_E164_PATTERN,
|
||||
base_score = 0.30,
|
||||
context_keywords = PHONE_CONTEXT,
|
||||
compliance_frameworks = ["GDPR",
|
||||
"CCPA"],
|
||||
),
|
||||
DetectionRule(
|
||||
rule_id = "PII_PASSPORT_US",
|
||||
rule_name = "US Passport Number",
|
||||
pattern = PASSPORT_US_PATTERN,
|
||||
base_score = 0.15,
|
||||
context_keywords = PASSPORT_CONTEXT,
|
||||
compliance_frameworks = ["GDPR",
|
||||
"CCPA"],
|
||||
),
|
||||
DetectionRule(
|
||||
rule_id = "PII_PASSPORT_UK",
|
||||
rule_name = "UK Passport Number",
|
||||
pattern = PASSPORT_UK_PATTERN,
|
||||
base_score = 0.10,
|
||||
context_keywords = PASSPORT_CONTEXT,
|
||||
compliance_frameworks = ["GDPR"],
|
||||
),
|
||||
DetectionRule(
|
||||
rule_id = "PII_IPV4",
|
||||
rule_name = "IPv4 Address",
|
||||
pattern = IPV4_PATTERN,
|
||||
base_score = 0.15,
|
||||
context_keywords = [],
|
||||
compliance_frameworks = ["GDPR"],
|
||||
),
|
||||
DetectionRule(
|
||||
rule_id = "PII_DRIVERS_LICENSE",
|
||||
rule_name = "US Driver's License (CA)",
|
||||
pattern = DRIVERS_LICENSE_CA_PATTERN,
|
||||
base_score = 0.10,
|
||||
context_keywords = DRIVERS_LICENSE_CONTEXT,
|
||||
compliance_frameworks = ["CCPA",
|
||||
"HIPAA"],
|
||||
),
|
||||
DetectionRule(
|
||||
rule_id = "PII_DRIVERS_LICENSE_FL",
|
||||
rule_name = "US Driver's License (FL)",
|
||||
pattern = DRIVERS_LICENSE_FL_PATTERN,
|
||||
base_score = 0.10,
|
||||
context_keywords = DRIVERS_LICENSE_CONTEXT,
|
||||
compliance_frameworks = ["CCPA",
|
||||
"HIPAA"],
|
||||
),
|
||||
DetectionRule(
|
||||
rule_id = "PII_DRIVERS_LICENSE_IL",
|
||||
rule_name = "US Driver's License (IL)",
|
||||
pattern = DRIVERS_LICENSE_IL_PATTERN,
|
||||
base_score = 0.10,
|
||||
context_keywords = DRIVERS_LICENSE_CONTEXT,
|
||||
compliance_frameworks = ["CCPA",
|
||||
"HIPAA"],
|
||||
),
|
||||
]
|
||||
|
|
@ -0,0 +1,146 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
engine.py
|
||||
"""
|
||||
|
||||
|
||||
import structlog
|
||||
|
||||
from dlp_scanner.config import ScanConfig
|
||||
from dlp_scanner.constants import OutputFormat
|
||||
from dlp_scanner.detectors.registry import (
|
||||
DetectorRegistry,
|
||||
)
|
||||
from dlp_scanner.models import ScanResult
|
||||
from dlp_scanner.reporters.console import (
|
||||
ConsoleReporter,
|
||||
)
|
||||
from dlp_scanner.reporters.csv_report import (
|
||||
CsvReporter,
|
||||
)
|
||||
from dlp_scanner.reporters.json_report import (
|
||||
JsonReporter,
|
||||
)
|
||||
from dlp_scanner.reporters.sarif import SarifReporter
|
||||
from dlp_scanner.scanners.db_scanner import (
|
||||
DatabaseScanner,
|
||||
)
|
||||
from dlp_scanner.scanners.file_scanner import (
|
||||
FileScanner,
|
||||
)
|
||||
from dlp_scanner.scanners.network_scanner import (
|
||||
NetworkScanner,
|
||||
)
|
||||
|
||||
|
||||
log = structlog.get_logger()
|
||||
|
||||
REPORTER_MAP: dict[str,
|
||||
type] = {
|
||||
"console": ConsoleReporter,
|
||||
"json": JsonReporter,
|
||||
"sarif": SarifReporter,
|
||||
"csv": CsvReporter,
|
||||
}
|
||||
|
||||
|
||||
class ScanEngine:
|
||||
"""
|
||||
Orchestrates the full scan pipeline
|
||||
"""
|
||||
def __init__(self, config: ScanConfig) -> None:
|
||||
self._config = config
|
||||
detection = config.detection
|
||||
allowlist_vals = detection.allowlists.values
|
||||
self._registry = DetectorRegistry(
|
||||
enable_patterns = detection.enable_rules,
|
||||
disable_patterns = detection.disable_rules,
|
||||
allowlist_values = (
|
||||
frozenset(allowlist_vals) if allowlist_vals else None
|
||||
),
|
||||
context_window_tokens = (detection.context_window_tokens),
|
||||
)
|
||||
|
||||
def scan_files(self, target: str) -> ScanResult:
|
||||
"""
|
||||
Scan filesystem target for sensitive data
|
||||
"""
|
||||
scanner = FileScanner(self._config, self._registry)
|
||||
result = scanner.scan(target)
|
||||
log.info(
|
||||
"file_scan_complete",
|
||||
target = target,
|
||||
findings = len(result.findings),
|
||||
targets = result.targets_scanned,
|
||||
)
|
||||
return result
|
||||
|
||||
def scan_database(self, target: str) -> ScanResult:
|
||||
"""
|
||||
Scan database target for sensitive data
|
||||
"""
|
||||
scanner = DatabaseScanner(self._config, self._registry)
|
||||
result = scanner.scan(target)
|
||||
log.info(
|
||||
"database_scan_complete",
|
||||
target = target,
|
||||
findings = len(result.findings),
|
||||
targets = result.targets_scanned,
|
||||
)
|
||||
return result
|
||||
|
||||
def scan_network(self, target: str) -> ScanResult:
|
||||
"""
|
||||
Scan network capture file for sensitive data
|
||||
"""
|
||||
scanner = NetworkScanner(self._config, self._registry)
|
||||
result = scanner.scan(target)
|
||||
log.info(
|
||||
"network_scan_complete",
|
||||
target = target,
|
||||
findings = len(result.findings),
|
||||
targets = result.targets_scanned,
|
||||
)
|
||||
return result
|
||||
|
||||
def generate_report(
|
||||
self,
|
||||
result: ScanResult,
|
||||
output_format: OutputFormat | None = None,
|
||||
) -> str:
|
||||
"""
|
||||
Generate report string in the requested format
|
||||
"""
|
||||
fmt = output_format or self._config.output.format
|
||||
reporter_cls = REPORTER_MAP[fmt]
|
||||
reporter = reporter_cls()
|
||||
output: str = reporter.generate(result)
|
||||
return output
|
||||
|
||||
def display_console(
|
||||
self,
|
||||
result: ScanResult,
|
||||
) -> None:
|
||||
"""
|
||||
Display Rich-formatted results to console
|
||||
"""
|
||||
reporter = ConsoleReporter()
|
||||
reporter.display(result)
|
||||
|
||||
def write_report(
|
||||
self,
|
||||
result: ScanResult,
|
||||
output_path: str,
|
||||
output_format: OutputFormat | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Generate report and write to file
|
||||
"""
|
||||
content = self.generate_report(result, output_format)
|
||||
with open(output_path, "w") as f:
|
||||
f.write(content)
|
||||
log.info(
|
||||
"report_written",
|
||||
path = output_path,
|
||||
format = output_format or self._config.output.format,
|
||||
)
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
__init__.py
|
||||
"""
|
||||
|
|
@ -0,0 +1,191 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
archive.py
|
||||
"""
|
||||
|
||||
|
||||
import tarfile
|
||||
import zipfile
|
||||
|
||||
import structlog
|
||||
|
||||
from dlp_scanner.constants import (
|
||||
MAX_ARCHIVE_DEPTH,
|
||||
MAX_ARCHIVE_MEMBER_SIZE_MB,
|
||||
ZIP_BOMB_RATIO_THRESHOLD,
|
||||
)
|
||||
from dlp_scanner.models import Location, TextChunk
|
||||
|
||||
|
||||
log = structlog.get_logger()
|
||||
|
||||
ARCHIVE_EXTENSIONS: frozenset[str] = frozenset(
|
||||
{
|
||||
".zip",
|
||||
".tar",
|
||||
".tar.gz",
|
||||
".tgz",
|
||||
".tar.bz2",
|
||||
}
|
||||
)
|
||||
|
||||
MAX_MEMBER_BYTES: int = MAX_ARCHIVE_MEMBER_SIZE_MB * 1024 * 1024
|
||||
|
||||
|
||||
class ArchiveExtractor:
|
||||
"""
|
||||
Extracts text content from archive files with security guards
|
||||
"""
|
||||
@property
|
||||
def supported_extensions(self) -> frozenset[str]:
|
||||
"""
|
||||
File extensions this extractor handles
|
||||
"""
|
||||
return ARCHIVE_EXTENSIONS
|
||||
|
||||
def extract(
|
||||
self,
|
||||
path: str,
|
||||
depth: int = 0,
|
||||
) -> list[TextChunk]:
|
||||
"""
|
||||
Extract text from archive members
|
||||
"""
|
||||
if depth >= MAX_ARCHIVE_DEPTH:
|
||||
log.warning(
|
||||
"archive_depth_exceeded",
|
||||
path = path,
|
||||
depth = depth,
|
||||
)
|
||||
return []
|
||||
|
||||
if path.endswith(".zip"):
|
||||
return self._extract_zip(path, depth)
|
||||
|
||||
if any(path.endswith(ext)
|
||||
for ext in (".tar", ".tar.gz", ".tgz", ".tar.bz2")):
|
||||
return self._extract_tar(path, depth)
|
||||
|
||||
return []
|
||||
|
||||
def _extract_zip(self, path: str, depth: int) -> list[TextChunk]:
|
||||
"""
|
||||
Extract from ZIP with bomb and traversal protection
|
||||
"""
|
||||
chunks: list[TextChunk] = []
|
||||
|
||||
try:
|
||||
with zipfile.ZipFile(path, "r") as zf:
|
||||
for info in zf.infolist():
|
||||
if not self._is_safe_zip_member(info):
|
||||
continue
|
||||
|
||||
data = zf.read(info.filename)
|
||||
if not data:
|
||||
continue
|
||||
|
||||
try:
|
||||
text = data.decode("utf-8", errors = "replace")
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
if text.strip():
|
||||
chunks.append(
|
||||
TextChunk(
|
||||
text = text,
|
||||
location = Location(
|
||||
source_type = "archive",
|
||||
uri = f"{path}!{info.filename}",
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
except Exception:
|
||||
log.warning("zip_extract_failed", path = path)
|
||||
|
||||
return chunks
|
||||
|
||||
def _extract_tar(self, path: str, depth: int) -> list[TextChunk]:
|
||||
"""
|
||||
Extract from TAR with traversal protection
|
||||
"""
|
||||
chunks: list[TextChunk] = []
|
||||
|
||||
try:
|
||||
with tarfile.open(path) as tf:
|
||||
for member in tf.getmembers():
|
||||
if not member.isfile():
|
||||
continue
|
||||
|
||||
if not self._is_safe_tar_member(member):
|
||||
continue
|
||||
|
||||
if member.size > MAX_MEMBER_BYTES:
|
||||
continue
|
||||
|
||||
extracted = tf.extractfile(member)
|
||||
if extracted is None:
|
||||
continue
|
||||
|
||||
data = extracted.read()
|
||||
try:
|
||||
text = data.decode("utf-8", errors = "replace")
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
if text.strip():
|
||||
chunks.append(
|
||||
TextChunk(
|
||||
text = text,
|
||||
location = Location(
|
||||
source_type = "archive",
|
||||
uri = f"{path}!{member.name}",
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
except Exception:
|
||||
log.warning("tar_extract_failed", path = path)
|
||||
|
||||
return chunks
|
||||
|
||||
def _is_safe_zip_member(self, info: zipfile.ZipInfo) -> bool:
|
||||
"""
|
||||
Check a ZIP member for path traversal and bomb indicators
|
||||
"""
|
||||
if ".." in info.filename or info.filename.startswith("/"):
|
||||
log.warning(
|
||||
"zip_path_traversal_blocked",
|
||||
filename = info.filename,
|
||||
)
|
||||
return False
|
||||
|
||||
if "\x00" in info.filename:
|
||||
return False
|
||||
|
||||
if info.file_size > MAX_MEMBER_BYTES:
|
||||
return False
|
||||
|
||||
if (info.compress_size > 0 and info.file_size / info.compress_size
|
||||
> ZIP_BOMB_RATIO_THRESHOLD):
|
||||
log.warning(
|
||||
"zip_bomb_detected",
|
||||
filename = info.filename,
|
||||
ratio = info.file_size / info.compress_size,
|
||||
)
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def _is_safe_tar_member(self, member: tarfile.TarInfo) -> bool:
|
||||
"""
|
||||
Check a TAR member for path traversal
|
||||
"""
|
||||
if ".." in member.name or member.name.startswith("/"):
|
||||
log.warning(
|
||||
"tar_path_traversal_blocked",
|
||||
filename = member.name,
|
||||
)
|
||||
return False
|
||||
|
||||
return not (member.issym() or member.islnk())
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
base.py
|
||||
"""
|
||||
|
||||
|
||||
from typing import Protocol
|
||||
|
||||
from dlp_scanner.models import TextChunk
|
||||
|
||||
|
||||
class Extractor(Protocol):
|
||||
"""
|
||||
Protocol for text extraction from different file formats
|
||||
"""
|
||||
def extract(self, path: str) -> list[TextChunk]:
|
||||
"""
|
||||
Extract text chunks from a file at the given path
|
||||
"""
|
||||
...
|
||||
|
||||
@property
|
||||
def supported_extensions(self) -> frozenset[str]:
|
||||
"""
|
||||
File extensions this extractor handles
|
||||
"""
|
||||
...
|
||||
|
|
@ -0,0 +1,123 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
email.py
|
||||
"""
|
||||
|
||||
|
||||
import email as email_lib
|
||||
from email import policy
|
||||
|
||||
import structlog
|
||||
|
||||
from dlp_scanner.models import Location, TextChunk
|
||||
|
||||
|
||||
log = structlog.get_logger()
|
||||
|
||||
EMAIL_EXTENSIONS: frozenset[str] = frozenset({
|
||||
".eml",
|
||||
".msg",
|
||||
})
|
||||
|
||||
|
||||
class EmlExtractor:
|
||||
"""
|
||||
Extracts text from RFC 2822 EML files
|
||||
"""
|
||||
@property
|
||||
def supported_extensions(self) -> frozenset[str]:
|
||||
"""
|
||||
File extensions this extractor handles
|
||||
"""
|
||||
return frozenset({".eml"})
|
||||
|
||||
def extract(self, path: str) -> list[TextChunk]:
|
||||
"""
|
||||
Parse EML and extract headers and body text
|
||||
"""
|
||||
chunks: list[TextChunk] = []
|
||||
|
||||
try:
|
||||
with open(path, "rb") as f:
|
||||
msg = email_lib.message_from_binary_file(
|
||||
f,
|
||||
policy = policy.default
|
||||
)
|
||||
|
||||
parts: list[str] = []
|
||||
|
||||
for header in ("From", "To", "Cc", "Subject"):
|
||||
value = msg.get(header)
|
||||
if value:
|
||||
parts.append(f"{header}: {value}")
|
||||
|
||||
body = msg.get_body(preferencelist = ("plain", "html"))
|
||||
if body is not None:
|
||||
content = body.get_content()
|
||||
if content:
|
||||
parts.append(content)
|
||||
|
||||
if parts:
|
||||
chunks.append(
|
||||
TextChunk(
|
||||
text = "\n".join(parts),
|
||||
location = Location(
|
||||
source_type = "file",
|
||||
uri = path,
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
except Exception:
|
||||
log.warning("eml_extract_failed", path = path)
|
||||
|
||||
return chunks
|
||||
|
||||
|
||||
class MsgExtractor:
|
||||
"""
|
||||
Extracts text from Outlook MSG files
|
||||
"""
|
||||
@property
|
||||
def supported_extensions(self) -> frozenset[str]:
|
||||
"""
|
||||
File extensions this extractor handles
|
||||
"""
|
||||
return frozenset({".msg"})
|
||||
|
||||
def extract(self, path: str) -> list[TextChunk]:
|
||||
"""
|
||||
Parse MSG and extract headers and body text
|
||||
"""
|
||||
import extract_msg
|
||||
|
||||
chunks: list[TextChunk] = []
|
||||
|
||||
try:
|
||||
with extract_msg.Message(path) as msg:
|
||||
parts: list[str] = []
|
||||
|
||||
if msg.sender:
|
||||
parts.append(f"From: {msg.sender}")
|
||||
if msg.to:
|
||||
parts.append(f"To: {msg.to}")
|
||||
if msg.subject:
|
||||
parts.append(f"Subject: {msg.subject}")
|
||||
if msg.body:
|
||||
parts.append(msg.body)
|
||||
|
||||
if parts:
|
||||
chunks.append(
|
||||
TextChunk(
|
||||
text = "\n".join(parts),
|
||||
location = Location(
|
||||
source_type = "file",
|
||||
uri = path,
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
except Exception:
|
||||
log.warning("msg_extract_failed", path = path)
|
||||
|
||||
return chunks
|
||||
|
|
@ -0,0 +1,178 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
office.py
|
||||
"""
|
||||
|
||||
|
||||
import structlog
|
||||
|
||||
from dlp_scanner.models import Location, TextChunk
|
||||
|
||||
|
||||
log = structlog.get_logger()
|
||||
|
||||
DOCX_EXTENSIONS: frozenset[str] = frozenset({".docx"})
|
||||
XLSX_EXTENSIONS: frozenset[str] = frozenset({".xlsx"})
|
||||
XLS_EXTENSIONS: frozenset[str] = frozenset({".xls"})
|
||||
OFFICE_EXTENSIONS: frozenset[str] = (
|
||||
DOCX_EXTENSIONS | XLSX_EXTENSIONS | XLS_EXTENSIONS
|
||||
)
|
||||
|
||||
|
||||
class DocxExtractor:
|
||||
"""
|
||||
Extracts text from DOCX files
|
||||
"""
|
||||
@property
|
||||
def supported_extensions(self) -> frozenset[str]:
|
||||
"""
|
||||
File extensions this extractor handles
|
||||
"""
|
||||
return DOCX_EXTENSIONS
|
||||
|
||||
def extract(self, path: str) -> list[TextChunk]:
|
||||
"""
|
||||
Extract text from paragraphs, tables, and metadata
|
||||
"""
|
||||
from docx import Document
|
||||
|
||||
chunks: list[TextChunk] = []
|
||||
|
||||
try:
|
||||
doc = Document(path)
|
||||
paragraphs: list[str] = []
|
||||
|
||||
for para in doc.paragraphs:
|
||||
if para.text.strip():
|
||||
paragraphs.append(para.text)
|
||||
|
||||
for table in doc.tables:
|
||||
for row in table.rows:
|
||||
cells = [
|
||||
cell.text
|
||||
for cell in row.cells
|
||||
if cell.text.strip()
|
||||
]
|
||||
if cells:
|
||||
paragraphs.append(" | ".join(cells))
|
||||
|
||||
if doc.core_properties.author:
|
||||
paragraphs.append(f"Author: {doc.core_properties.author}")
|
||||
if doc.core_properties.title:
|
||||
paragraphs.append(f"Title: {doc.core_properties.title}")
|
||||
|
||||
if paragraphs:
|
||||
chunks.append(
|
||||
TextChunk(
|
||||
text = "\n".join(paragraphs),
|
||||
location = Location(
|
||||
source_type = "file",
|
||||
uri = path,
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
except Exception:
|
||||
log.warning("docx_extract_failed", path = path)
|
||||
|
||||
return chunks
|
||||
|
||||
|
||||
class XlsxExtractor:
|
||||
"""
|
||||
Extracts text from XLSX files using openpyxl
|
||||
"""
|
||||
@property
|
||||
def supported_extensions(self) -> frozenset[str]:
|
||||
"""
|
||||
File extensions this extractor handles
|
||||
"""
|
||||
return XLSX_EXTENSIONS
|
||||
|
||||
def extract(self, path: str) -> list[TextChunk]:
|
||||
"""
|
||||
Extract text from all sheets and cells
|
||||
"""
|
||||
from openpyxl import load_workbook
|
||||
|
||||
chunks: list[TextChunk] = []
|
||||
|
||||
try:
|
||||
wb = load_workbook(
|
||||
path,
|
||||
read_only = True,
|
||||
data_only = True,
|
||||
)
|
||||
for sheet in wb.worksheets:
|
||||
rows: list[str] = []
|
||||
for row in sheet.iter_rows(values_only = True):
|
||||
cell_values = [str(c) for c in row if c is not None]
|
||||
if cell_values:
|
||||
rows.append(" | ".join(cell_values))
|
||||
|
||||
if rows:
|
||||
chunks.append(
|
||||
TextChunk(
|
||||
text = "\n".join(rows),
|
||||
location = Location(
|
||||
source_type = "file",
|
||||
uri = path,
|
||||
sheet_name = sheet.title,
|
||||
),
|
||||
)
|
||||
)
|
||||
wb.close()
|
||||
except Exception:
|
||||
log.warning("xlsx_extract_failed", path = path)
|
||||
|
||||
return chunks
|
||||
|
||||
|
||||
class XlsExtractor:
|
||||
"""
|
||||
Extracts text from legacy XLS files using xlrd
|
||||
"""
|
||||
@property
|
||||
def supported_extensions(self) -> frozenset[str]:
|
||||
"""
|
||||
File extensions this extractor handles
|
||||
"""
|
||||
return XLS_EXTENSIONS
|
||||
|
||||
def extract(self, path: str) -> list[TextChunk]:
|
||||
"""
|
||||
Extract text from legacy Excel workbooks
|
||||
"""
|
||||
import xlrd
|
||||
|
||||
chunks: list[TextChunk] = []
|
||||
|
||||
try:
|
||||
wb = xlrd.open_workbook(path)
|
||||
for sheet in wb.sheets():
|
||||
rows: list[str] = []
|
||||
for row_idx in range(sheet.nrows):
|
||||
cell_values = [
|
||||
str(sheet.cell_value(row_idx,
|
||||
col))
|
||||
for col in range(sheet.ncols)
|
||||
if sheet.cell_value(row_idx, col)
|
||||
]
|
||||
if cell_values:
|
||||
rows.append(" | ".join(cell_values))
|
||||
|
||||
if rows:
|
||||
chunks.append(
|
||||
TextChunk(
|
||||
text = "\n".join(rows),
|
||||
location = Location(
|
||||
source_type = "file",
|
||||
uri = path,
|
||||
sheet_name = sheet.name,
|
||||
),
|
||||
)
|
||||
)
|
||||
except Exception:
|
||||
log.warning("xls_extract_failed", path = path)
|
||||
|
||||
return chunks
|
||||
|
|
@ -0,0 +1,56 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
pdf.py
|
||||
"""
|
||||
|
||||
|
||||
import structlog
|
||||
|
||||
from dlp_scanner.models import Location, TextChunk
|
||||
|
||||
|
||||
log = structlog.get_logger()
|
||||
|
||||
PDF_EXTENSIONS: frozenset[str] = frozenset({".pdf"})
|
||||
|
||||
|
||||
class PDFExtractor:
|
||||
"""
|
||||
Extracts text from PDF files using PyMuPDF
|
||||
"""
|
||||
@property
|
||||
def supported_extensions(self) -> frozenset[str]:
|
||||
"""
|
||||
File extensions this extractor handles
|
||||
"""
|
||||
return PDF_EXTENSIONS
|
||||
|
||||
def extract(self, path: str) -> list[TextChunk]:
|
||||
"""
|
||||
Extract text from each page of a PDF
|
||||
"""
|
||||
import fitz
|
||||
|
||||
chunks: list[TextChunk] = []
|
||||
|
||||
try:
|
||||
doc = fitz.open(path)
|
||||
for page_num in range(len(doc)):
|
||||
page = doc[page_num]
|
||||
text = page.get_text("text")
|
||||
if text.strip():
|
||||
chunks.append(
|
||||
TextChunk(
|
||||
text = text,
|
||||
location = Location(
|
||||
source_type = "file",
|
||||
uri = path,
|
||||
line = page_num + 1,
|
||||
),
|
||||
)
|
||||
)
|
||||
doc.close()
|
||||
except Exception:
|
||||
log.warning("pdf_extract_failed", path = path)
|
||||
|
||||
return chunks
|
||||
|
|
@ -0,0 +1,112 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
plaintext.py
|
||||
"""
|
||||
|
||||
|
||||
import structlog
|
||||
|
||||
from dlp_scanner.models import Location, TextChunk
|
||||
|
||||
|
||||
log = structlog.get_logger()
|
||||
|
||||
PLAINTEXT_EXTENSIONS: frozenset[str] = frozenset(
|
||||
{
|
||||
".txt",
|
||||
".log",
|
||||
".cfg",
|
||||
".ini",
|
||||
".conf",
|
||||
".toml",
|
||||
".md",
|
||||
".rst",
|
||||
".html",
|
||||
".htm",
|
||||
".tsv",
|
||||
".env",
|
||||
".sh",
|
||||
".bat",
|
||||
".ps1",
|
||||
".py",
|
||||
".js",
|
||||
".ts",
|
||||
".go",
|
||||
".rb",
|
||||
".java",
|
||||
".c",
|
||||
".cpp",
|
||||
".h",
|
||||
".hpp",
|
||||
".rs",
|
||||
".tf",
|
||||
".hcl",
|
||||
}
|
||||
)
|
||||
|
||||
CHUNK_MAX_LINES: int = 500
|
||||
|
||||
|
||||
class PlaintextExtractor:
|
||||
"""
|
||||
Extracts text from plaintext and source code files
|
||||
"""
|
||||
@property
|
||||
def supported_extensions(self) -> frozenset[str]:
|
||||
"""
|
||||
File extensions this extractor handles
|
||||
"""
|
||||
return PLAINTEXT_EXTENSIONS
|
||||
|
||||
def extract(self, path: str) -> list[TextChunk]:
|
||||
"""
|
||||
Read a text file and return chunks
|
||||
"""
|
||||
chunks: list[TextChunk] = []
|
||||
|
||||
try:
|
||||
with open(
|
||||
path,
|
||||
encoding = "utf-8",
|
||||
errors = "replace",
|
||||
) as f:
|
||||
lines: list[str] = []
|
||||
line_number = 1
|
||||
chunk_start = 1
|
||||
|
||||
for line in f:
|
||||
lines.append(line)
|
||||
if len(lines) >= CHUNK_MAX_LINES:
|
||||
chunks.append(
|
||||
TextChunk(
|
||||
text = "".join(lines),
|
||||
location = Location(
|
||||
source_type = "file",
|
||||
uri = path,
|
||||
line = chunk_start,
|
||||
),
|
||||
)
|
||||
)
|
||||
chunk_start = line_number + 1
|
||||
lines = []
|
||||
line_number += 1
|
||||
|
||||
if lines:
|
||||
chunks.append(
|
||||
TextChunk(
|
||||
text = "".join(lines),
|
||||
location = Location(
|
||||
source_type = "file",
|
||||
uri = path,
|
||||
line = chunk_start,
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
except OSError:
|
||||
log.warning(
|
||||
"file_read_failed",
|
||||
path = path,
|
||||
)
|
||||
|
||||
return chunks
|
||||
|
|
@ -0,0 +1,327 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
structured.py
|
||||
"""
|
||||
|
||||
|
||||
import csv
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
import structlog
|
||||
|
||||
from dlp_scanner.models import Location, TextChunk
|
||||
|
||||
|
||||
log = structlog.get_logger()
|
||||
|
||||
|
||||
class CsvExtractor:
|
||||
"""
|
||||
Extracts text from CSV and TSV files
|
||||
"""
|
||||
@property
|
||||
def supported_extensions(self) -> frozenset[str]:
|
||||
"""
|
||||
File extensions this extractor handles
|
||||
"""
|
||||
return frozenset({".csv", ".tsv"})
|
||||
|
||||
def extract(self, path: str) -> list[TextChunk]:
|
||||
"""
|
||||
Read CSV row by row and concatenate cell values
|
||||
"""
|
||||
chunks: list[TextChunk] = []
|
||||
|
||||
try:
|
||||
with open(
|
||||
path,
|
||||
newline = "",
|
||||
encoding = "utf-8-sig",
|
||||
) as f:
|
||||
dialect = csv.Sniffer().sniff(f.read(4096))
|
||||
f.seek(0)
|
||||
reader = csv.reader(f, dialect)
|
||||
rows: list[str] = []
|
||||
|
||||
for _row_num, row in enumerate(reader, 1):
|
||||
cells = [c for c in row if c.strip()]
|
||||
if cells:
|
||||
rows.append(" | ".join(cells))
|
||||
|
||||
if rows:
|
||||
chunks.append(
|
||||
TextChunk(
|
||||
text = "\n".join(rows),
|
||||
location = Location(
|
||||
source_type = "file",
|
||||
uri = path,
|
||||
line = 1,
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
except Exception:
|
||||
log.warning("csv_extract_failed", path = path)
|
||||
|
||||
return chunks
|
||||
|
||||
|
||||
class JsonExtractor:
|
||||
"""
|
||||
Extracts text values from JSON files
|
||||
"""
|
||||
@property
|
||||
def supported_extensions(self) -> frozenset[str]:
|
||||
"""
|
||||
File extensions this extractor handles
|
||||
"""
|
||||
return frozenset({".json"})
|
||||
|
||||
def extract(self, path: str) -> list[TextChunk]:
|
||||
"""
|
||||
Parse JSON and extract all string values recursively
|
||||
"""
|
||||
chunks: list[TextChunk] = []
|
||||
|
||||
try:
|
||||
with open(path, encoding = "utf-8") as f:
|
||||
data = json.load(f)
|
||||
|
||||
strings = _extract_json_strings(data)
|
||||
if strings:
|
||||
chunks.append(
|
||||
TextChunk(
|
||||
text = "\n".join(strings),
|
||||
location = Location(
|
||||
source_type = "file",
|
||||
uri = path,
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
except Exception:
|
||||
log.warning("json_extract_failed", path = path)
|
||||
|
||||
return chunks
|
||||
|
||||
|
||||
class XmlExtractor:
|
||||
"""
|
||||
Extracts text from XML files using defusedxml
|
||||
"""
|
||||
@property
|
||||
def supported_extensions(self) -> frozenset[str]:
|
||||
"""
|
||||
File extensions this extractor handles
|
||||
"""
|
||||
return frozenset({".xml"})
|
||||
|
||||
def extract(self, path: str) -> list[TextChunk]:
|
||||
"""
|
||||
Parse XML safely and extract all text content
|
||||
"""
|
||||
import defusedxml.ElementTree as ET
|
||||
|
||||
chunks: list[TextChunk] = []
|
||||
|
||||
try:
|
||||
tree = ET.parse(path)
|
||||
root = tree.getroot()
|
||||
texts: list[str] = []
|
||||
|
||||
for elem in root.iter():
|
||||
if elem.text and elem.text.strip():
|
||||
texts.append(elem.text.strip())
|
||||
if elem.tail and elem.tail.strip():
|
||||
texts.append(elem.tail.strip())
|
||||
for attr_val in elem.attrib.values():
|
||||
if attr_val.strip():
|
||||
texts.append(attr_val.strip())
|
||||
|
||||
if texts:
|
||||
chunks.append(
|
||||
TextChunk(
|
||||
text = "\n".join(texts),
|
||||
location = Location(
|
||||
source_type = "file",
|
||||
uri = path,
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
except Exception:
|
||||
log.warning("xml_extract_failed", path = path)
|
||||
|
||||
return chunks
|
||||
|
||||
|
||||
class YamlExtractor:
|
||||
"""
|
||||
Extracts text from YAML files
|
||||
"""
|
||||
@property
|
||||
def supported_extensions(self) -> frozenset[str]:
|
||||
"""
|
||||
File extensions this extractor handles
|
||||
"""
|
||||
return frozenset({".yaml", ".yml"})
|
||||
|
||||
def extract(self, path: str) -> list[TextChunk]:
|
||||
"""
|
||||
Parse YAML safely and extract string values
|
||||
"""
|
||||
from ruamel.yaml import YAML
|
||||
|
||||
chunks: list[TextChunk] = []
|
||||
|
||||
try:
|
||||
yaml = YAML(typ = "safe")
|
||||
with open(path) as f:
|
||||
data = yaml.load(f)
|
||||
|
||||
if data:
|
||||
strings = _extract_json_strings(data)
|
||||
if strings:
|
||||
chunks.append(
|
||||
TextChunk(
|
||||
text = "\n".join(strings),
|
||||
location = Location(
|
||||
source_type = "file",
|
||||
uri = path,
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
except Exception:
|
||||
log.warning("yaml_extract_failed", path = path)
|
||||
|
||||
return chunks
|
||||
|
||||
|
||||
class ParquetExtractor:
|
||||
"""
|
||||
Extracts text from Parquet files
|
||||
"""
|
||||
@property
|
||||
def supported_extensions(self) -> frozenset[str]:
|
||||
"""
|
||||
File extensions this extractor handles
|
||||
"""
|
||||
return frozenset({".parquet"})
|
||||
|
||||
def extract(self, path: str) -> list[TextChunk]:
|
||||
"""
|
||||
Read Parquet file and extract string columns
|
||||
"""
|
||||
import pyarrow.parquet as pq
|
||||
|
||||
chunks: list[TextChunk] = []
|
||||
|
||||
try:
|
||||
pf = pq.ParquetFile(path)
|
||||
schema = pf.schema_arrow
|
||||
|
||||
string_cols = [
|
||||
field.name for field in schema if str(field.type) in (
|
||||
"string",
|
||||
"large_string",
|
||||
"utf8",
|
||||
"large_utf8",)
|
||||
]
|
||||
|
||||
if not string_cols:
|
||||
return chunks
|
||||
|
||||
for batch in pf.iter_batches(
|
||||
batch_size = 5000,
|
||||
columns = string_cols,
|
||||
):
|
||||
rows: list[str] = []
|
||||
table_dict = batch.to_pydict()
|
||||
for col_name, values in table_dict.items():
|
||||
for val in values:
|
||||
if val is not None and str(val).strip():
|
||||
rows.append(f"{col_name}: {val}")
|
||||
if rows:
|
||||
chunks.append(
|
||||
TextChunk(
|
||||
text = "\n".join(rows),
|
||||
location = Location(
|
||||
source_type = "file",
|
||||
uri = path,
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
except Exception:
|
||||
log.warning("parquet_extract_failed", path = path)
|
||||
|
||||
return chunks
|
||||
|
||||
|
||||
class AvroExtractor:
|
||||
"""
|
||||
Extracts text from Avro files
|
||||
"""
|
||||
@property
|
||||
def supported_extensions(self) -> frozenset[str]:
|
||||
"""
|
||||
File extensions this extractor handles
|
||||
"""
|
||||
return frozenset({".avro"})
|
||||
|
||||
def extract(self, path: str) -> list[TextChunk]:
|
||||
"""
|
||||
Read Avro file and extract string fields
|
||||
"""
|
||||
from fastavro import reader
|
||||
|
||||
chunks: list[TextChunk] = []
|
||||
|
||||
try:
|
||||
with open(path, "rb") as f:
|
||||
rows: list[str] = []
|
||||
for record in reader(f):
|
||||
strings = _extract_json_strings(record)
|
||||
rows.extend(strings)
|
||||
|
||||
if rows:
|
||||
chunks.append(
|
||||
TextChunk(
|
||||
text = "\n".join(rows),
|
||||
location = Location(
|
||||
source_type = "file",
|
||||
uri = path,
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
except Exception:
|
||||
log.warning("avro_extract_failed", path = path)
|
||||
|
||||
return chunks
|
||||
|
||||
|
||||
def _extract_json_strings(
|
||||
data: Any,
|
||||
prefix: str = "",
|
||||
) -> list[str]:
|
||||
"""
|
||||
Recursively extract all string values from a JSON-like structure
|
||||
"""
|
||||
strings: list[str] = []
|
||||
|
||||
if isinstance(data, str):
|
||||
if data.strip():
|
||||
label = f"{prefix}: {data}" if prefix else data
|
||||
strings.append(label)
|
||||
elif isinstance(data, dict):
|
||||
for key, val in data.items():
|
||||
key_path = (f"{prefix}.{key}" if prefix else str(key))
|
||||
strings.extend(_extract_json_strings(val, key_path))
|
||||
elif isinstance(data, list):
|
||||
for item in data:
|
||||
strings.extend(_extract_json_strings(item, prefix))
|
||||
|
||||
return strings
|
||||
|
|
@ -0,0 +1,80 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
log.py
|
||||
"""
|
||||
|
||||
|
||||
import logging
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
import orjson
|
||||
import structlog
|
||||
|
||||
|
||||
def _orjson_serializer(
|
||||
data: Any,
|
||||
**_kwargs: Any,
|
||||
) -> str:
|
||||
"""
|
||||
Serialize log data using orjson for performance
|
||||
"""
|
||||
return orjson.dumps(data).decode("utf-8")
|
||||
|
||||
|
||||
def configure_logging(
|
||||
level: str = "INFO",
|
||||
json_output: bool = False,
|
||||
log_file: str = "",
|
||||
) -> None:
|
||||
"""
|
||||
Set up structlog with stdlib integration
|
||||
"""
|
||||
shared_processors: list[structlog.types.Processor] = [
|
||||
structlog.contextvars.merge_contextvars,
|
||||
structlog.stdlib.add_log_level,
|
||||
structlog.stdlib.add_logger_name,
|
||||
structlog.processors.TimeStamper(fmt = "iso"),
|
||||
structlog.stdlib.PositionalArgumentsFormatter(),
|
||||
structlog.processors.StackInfoRenderer(),
|
||||
]
|
||||
|
||||
if json_output:
|
||||
renderer: structlog.types.Processor = (
|
||||
structlog.processors.JSONRenderer(
|
||||
serializer = _orjson_serializer
|
||||
)
|
||||
)
|
||||
else:
|
||||
renderer = structlog.dev.ConsoleRenderer(colors = True)
|
||||
|
||||
structlog.configure(
|
||||
processors = [
|
||||
*shared_processors,
|
||||
structlog.stdlib.ProcessorFormatter.wrap_for_formatter,
|
||||
],
|
||||
logger_factory = structlog.stdlib.LoggerFactory(),
|
||||
wrapper_class = structlog.stdlib.BoundLogger,
|
||||
cache_logger_on_first_use = True,
|
||||
)
|
||||
|
||||
formatter = structlog.stdlib.ProcessorFormatter(
|
||||
foreign_pre_chain = shared_processors,
|
||||
processors = [
|
||||
structlog.stdlib.ProcessorFormatter.remove_processors_meta,
|
||||
renderer,
|
||||
],
|
||||
)
|
||||
|
||||
handler: logging.Handler = logging.StreamHandler(sys.stderr)
|
||||
handler.setFormatter(formatter)
|
||||
|
||||
root_logger = logging.getLogger()
|
||||
root_logger.handlers.clear()
|
||||
root_logger.addHandler(handler)
|
||||
root_logger.setLevel(getattr(logging, level.upper()))
|
||||
|
||||
if log_file:
|
||||
file_handler = logging.FileHandler(log_file)
|
||||
file_handler.setFormatter(formatter)
|
||||
root_logger.addHandler(file_handler)
|
||||
|
|
@ -0,0 +1,112 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
models.py
|
||||
"""
|
||||
|
||||
import uuid
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, UTC
|
||||
|
||||
from dlp_scanner.constants import Severity
|
||||
|
||||
|
||||
@dataclass(frozen = True, slots = True)
|
||||
class Location:
|
||||
"""
|
||||
Where a finding was detected
|
||||
"""
|
||||
source_type: str
|
||||
uri: str
|
||||
line: int | None = None
|
||||
column: int | None = None
|
||||
byte_offset: int | None = None
|
||||
table_name: str | None = None
|
||||
column_name: str | None = None
|
||||
sheet_name: str | None = None
|
||||
|
||||
|
||||
@dataclass(slots = True)
|
||||
class Finding:
|
||||
"""
|
||||
A fully scored and classified detection result
|
||||
"""
|
||||
finding_id: str = field(
|
||||
default_factory = lambda: uuid.uuid4().hex[: 12]
|
||||
)
|
||||
rule_id: str = ""
|
||||
rule_name: str = ""
|
||||
severity: Severity = "low"
|
||||
confidence: float = 0.0
|
||||
location: Location = field(
|
||||
default_factory = lambda: Location(
|
||||
source_type = "unknown",
|
||||
uri = "",)
|
||||
)
|
||||
redacted_snippet: str = ""
|
||||
compliance_frameworks: list[str] = field(default_factory = list)
|
||||
remediation: str = ""
|
||||
detected_at: datetime = field(
|
||||
default_factory = lambda: datetime.now(UTC)
|
||||
)
|
||||
|
||||
|
||||
@dataclass(slots = True)
|
||||
class ScanResult:
|
||||
"""
|
||||
Aggregated results from a complete scan run
|
||||
"""
|
||||
scan_id: str = field(default_factory = lambda: uuid.uuid4().hex[: 16])
|
||||
tool_version: str = "0.1.0"
|
||||
scan_started_at: datetime = field(
|
||||
default_factory = lambda: datetime.now(UTC)
|
||||
)
|
||||
scan_completed_at: datetime | None = None
|
||||
targets_scanned: int = 0
|
||||
findings: list[Finding] = field(default_factory = list)
|
||||
errors: list[str] = field(default_factory = list)
|
||||
|
||||
@property
|
||||
def findings_by_severity(self) -> dict[str, int]:
|
||||
"""
|
||||
Count findings grouped by severity level
|
||||
"""
|
||||
counts: dict[str,
|
||||
int] = {
|
||||
"critical": 0,
|
||||
"high": 0,
|
||||
"medium": 0,
|
||||
"low": 0,
|
||||
}
|
||||
for f in self.findings:
|
||||
counts[f.severity] = counts.get(f.severity, 0) + 1
|
||||
return counts
|
||||
|
||||
@property
|
||||
def findings_by_rule(self) -> dict[str, int]:
|
||||
"""
|
||||
Count findings grouped by rule ID
|
||||
"""
|
||||
counts: dict[str, int] = {}
|
||||
for f in self.findings:
|
||||
counts[f.rule_id] = counts.get(f.rule_id, 0) + 1
|
||||
return counts
|
||||
|
||||
@property
|
||||
def findings_by_framework(self) -> dict[str, int]:
|
||||
"""
|
||||
Count findings grouped by compliance framework
|
||||
"""
|
||||
counts: dict[str, int] = {}
|
||||
for f in self.findings:
|
||||
for fw in f.compliance_frameworks:
|
||||
counts[fw] = counts.get(fw, 0) + 1
|
||||
return counts
|
||||
|
||||
|
||||
@dataclass(frozen = True, slots = True)
|
||||
class TextChunk:
|
||||
"""
|
||||
A piece of extracted text with its source location
|
||||
"""
|
||||
text: str
|
||||
location: Location
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
__init__.py
|
||||
"""
|
||||
|
|
@ -0,0 +1,279 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
exfiltration.py
|
||||
"""
|
||||
|
||||
|
||||
import re
|
||||
from collections import defaultdict
|
||||
from dataclasses import dataclass
|
||||
|
||||
import structlog
|
||||
|
||||
from dlp_scanner.constants import (
|
||||
DEFAULT_DNS_ENTROPY_THRESHOLD,
|
||||
)
|
||||
from dlp_scanner.detectors.entropy import (
|
||||
shannon_entropy_str,
|
||||
)
|
||||
from dlp_scanner.network.protocols import DnsQuery
|
||||
|
||||
|
||||
log = structlog.get_logger()
|
||||
|
||||
DNS_LABEL_MAX_NORMAL: int = 50
|
||||
DNS_QNAME_MAX_NORMAL: int = 100
|
||||
TXT_VOLUME_THRESHOLD: float = 0.05
|
||||
|
||||
BASE64_PATTERN = re.compile(rb"[A-Za-z0-9+/]{40,}={0,2}")
|
||||
HEX_PATTERN = re.compile(rb"[0-9A-Fa-f]{64,}")
|
||||
|
||||
|
||||
@dataclass(frozen = True, slots = True)
|
||||
class ExfilIndicator:
|
||||
"""
|
||||
An indicator of potential data exfiltration
|
||||
"""
|
||||
|
||||
indicator_type: str
|
||||
description: str
|
||||
confidence: float
|
||||
source_ip: str
|
||||
dest_ip: str
|
||||
evidence: str
|
||||
|
||||
|
||||
class DnsExfilDetector:
|
||||
"""
|
||||
Detects DNS-based data exfiltration patterns
|
||||
"""
|
||||
def __init__(
|
||||
self,
|
||||
entropy_threshold: float = (DEFAULT_DNS_ENTROPY_THRESHOLD),
|
||||
) -> None:
|
||||
self._entropy_threshold = entropy_threshold
|
||||
self._indicators: list[ExfilIndicator] = []
|
||||
self._domain_txt_counts: dict[str, int] = defaultdict(int)
|
||||
self._domain_total_counts: dict[str, int] = defaultdict(int)
|
||||
|
||||
def analyze_query(
|
||||
self,
|
||||
query: DnsQuery,
|
||||
src_ip: str,
|
||||
dst_ip: str,
|
||||
) -> ExfilIndicator | None:
|
||||
"""
|
||||
Analyze a single DNS query for exfiltration
|
||||
"""
|
||||
name = query.name
|
||||
domain = _extract_base_domain(name)
|
||||
|
||||
self._domain_total_counts[domain] += 1
|
||||
if query.query_type == "TXT":
|
||||
self._domain_txt_counts[domain] += 1
|
||||
|
||||
indicator = self._check_label_length(name, src_ip, dst_ip)
|
||||
if indicator is not None:
|
||||
self._indicators.append(indicator)
|
||||
return indicator
|
||||
|
||||
indicator = self._check_subdomain_entropy(name, src_ip, dst_ip)
|
||||
if indicator is not None:
|
||||
self._indicators.append(indicator)
|
||||
return indicator
|
||||
|
||||
indicator = self._check_qname_length(name, src_ip, dst_ip)
|
||||
if indicator is not None:
|
||||
self._indicators.append(indicator)
|
||||
return indicator
|
||||
|
||||
return None
|
||||
|
||||
def check_txt_volume(
|
||||
self,
|
||||
) -> list[ExfilIndicator]:
|
||||
"""
|
||||
Check for suspicious TXT query volume ratios
|
||||
"""
|
||||
indicators: list[ExfilIndicator] = []
|
||||
|
||||
for domain, txt_count in (self._domain_txt_counts.items()):
|
||||
total = self._domain_total_counts.get(domain, 0)
|
||||
if total == 0:
|
||||
continue
|
||||
|
||||
ratio = txt_count / total
|
||||
if ratio > TXT_VOLUME_THRESHOLD:
|
||||
indicator = ExfilIndicator(
|
||||
indicator_type = "dns_txt_volume",
|
||||
description = (
|
||||
f"High TXT query ratio "
|
||||
f"({ratio:.1%}) for "
|
||||
f"{domain}"
|
||||
),
|
||||
confidence = min(0.90,
|
||||
0.50 + ratio),
|
||||
source_ip = "",
|
||||
dest_ip = "",
|
||||
evidence = (f"{txt_count} TXT / "
|
||||
f"{total} total"),
|
||||
)
|
||||
indicators.append(indicator)
|
||||
|
||||
self._indicators.extend(indicators)
|
||||
return indicators
|
||||
|
||||
def get_indicators(
|
||||
self,
|
||||
) -> list[ExfilIndicator]:
|
||||
"""
|
||||
Return all collected exfiltration indicators
|
||||
"""
|
||||
return list(self._indicators)
|
||||
|
||||
def _check_label_length(
|
||||
self,
|
||||
name: str,
|
||||
src_ip: str,
|
||||
dst_ip: str,
|
||||
) -> ExfilIndicator | None:
|
||||
"""
|
||||
Flag suspiciously long DNS labels
|
||||
"""
|
||||
for label in name.split("."):
|
||||
if len(label) > DNS_LABEL_MAX_NORMAL:
|
||||
return ExfilIndicator(
|
||||
indicator_type = ("dns_long_label"),
|
||||
description = (
|
||||
f"DNS label length "
|
||||
f"{len(label)} exceeds "
|
||||
f"normal threshold"
|
||||
),
|
||||
confidence = 0.75,
|
||||
source_ip = src_ip,
|
||||
dest_ip = dst_ip,
|
||||
evidence = name,
|
||||
)
|
||||
return None
|
||||
|
||||
def _check_subdomain_entropy(
|
||||
self,
|
||||
name: str,
|
||||
src_ip: str,
|
||||
dst_ip: str,
|
||||
) -> ExfilIndicator | None:
|
||||
"""
|
||||
Flag high-entropy subdomains suggesting tunneling
|
||||
"""
|
||||
parts = name.split(".")
|
||||
if len(parts) < 3:
|
||||
return None
|
||||
|
||||
subdomain = ".".join(parts[:-2])
|
||||
if not subdomain:
|
||||
return None
|
||||
|
||||
entropy = shannon_entropy_str(subdomain)
|
||||
if entropy > self._entropy_threshold:
|
||||
return ExfilIndicator(
|
||||
indicator_type = ("dns_high_entropy"),
|
||||
description = (
|
||||
f"High subdomain entropy "
|
||||
f"({entropy:.2f}) suggesting "
|
||||
f"DNS tunneling"
|
||||
),
|
||||
confidence = min(
|
||||
0.95,
|
||||
0.50 + (entropy - 3.0) * 0.15,
|
||||
),
|
||||
source_ip = src_ip,
|
||||
dest_ip = dst_ip,
|
||||
evidence = name,
|
||||
)
|
||||
return None
|
||||
|
||||
def _check_qname_length(
|
||||
self,
|
||||
name: str,
|
||||
src_ip: str,
|
||||
dst_ip: str,
|
||||
) -> ExfilIndicator | None:
|
||||
"""
|
||||
Flag excessively long QNAMEs
|
||||
"""
|
||||
if len(name) > DNS_QNAME_MAX_NORMAL:
|
||||
return ExfilIndicator(
|
||||
indicator_type = "dns_long_qname",
|
||||
description = (
|
||||
f"QNAME length {len(name)} "
|
||||
f"exceeds normal threshold"
|
||||
),
|
||||
confidence = 0.65,
|
||||
source_ip = src_ip,
|
||||
dest_ip = dst_ip,
|
||||
evidence = name,
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def detect_base64_payload(
|
||||
data: bytes,
|
||||
src_ip: str = "",
|
||||
dst_ip: str = "",
|
||||
) -> list[ExfilIndicator]:
|
||||
"""
|
||||
Detect base64 or hex-encoded data in payloads
|
||||
"""
|
||||
indicators: list[ExfilIndicator] = []
|
||||
|
||||
for m in BASE64_PATTERN.finditer(data):
|
||||
matched = m.group()
|
||||
indicators.append(
|
||||
ExfilIndicator(
|
||||
indicator_type = "base64_payload",
|
||||
description = (
|
||||
f"Base64-encoded data "
|
||||
f"({len(matched)} bytes) "
|
||||
f"in network payload"
|
||||
),
|
||||
confidence = 0.55,
|
||||
source_ip = src_ip,
|
||||
dest_ip = dst_ip,
|
||||
evidence = matched[: 80].decode(
|
||||
"ascii",
|
||||
errors = "replace"
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
for m in HEX_PATTERN.finditer(data):
|
||||
matched = m.group()
|
||||
indicators.append(
|
||||
ExfilIndicator(
|
||||
indicator_type = "hex_payload",
|
||||
description = (
|
||||
f"Hex-encoded data "
|
||||
f"({len(matched)} bytes) "
|
||||
f"in network payload"
|
||||
),
|
||||
confidence = 0.45,
|
||||
source_ip = src_ip,
|
||||
dest_ip = dst_ip,
|
||||
evidence = matched[: 80].decode(
|
||||
"ascii",
|
||||
errors = "replace"
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
return indicators
|
||||
|
||||
|
||||
def _extract_base_domain(name: str) -> str:
|
||||
"""
|
||||
Extract the registerable domain from a QNAME
|
||||
"""
|
||||
parts = name.rstrip(".").split(".")
|
||||
if len(parts) >= 2:
|
||||
return ".".join(parts[-2 :])
|
||||
return name
|
||||
|
|
@ -0,0 +1,126 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
flow_tracker.py
|
||||
"""
|
||||
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
import structlog
|
||||
|
||||
from dlp_scanner.network.pcap import PacketInfo
|
||||
|
||||
|
||||
log = structlog.get_logger()
|
||||
|
||||
FlowKey = tuple[str, str, int, int]
|
||||
|
||||
|
||||
@dataclass(slots = True)
|
||||
class FlowStats:
|
||||
"""
|
||||
Aggregated statistics for a network flow
|
||||
"""
|
||||
|
||||
src_ip: str = ""
|
||||
dst_ip: str = ""
|
||||
src_port: int = 0
|
||||
dst_port: int = 0
|
||||
protocol: str = ""
|
||||
packet_count: int = 0
|
||||
total_bytes: int = 0
|
||||
start_time: float = 0.0
|
||||
end_time: float = 0.0
|
||||
segments: list[tuple[int, bytes]] = field(default_factory = list)
|
||||
|
||||
|
||||
class FlowTracker:
|
||||
"""
|
||||
Tracks and reassembles network flows from packets
|
||||
"""
|
||||
def __init__(self) -> None:
|
||||
self._flows: dict[FlowKey, FlowStats] = {}
|
||||
|
||||
def add_packet(self, packet: PacketInfo) -> None:
|
||||
"""
|
||||
Add a packet to its corresponding flow
|
||||
"""
|
||||
key = make_flow_key(packet)
|
||||
flow = self._flows.get(key)
|
||||
|
||||
if flow is None:
|
||||
flow = FlowStats(
|
||||
src_ip = packet.src_ip,
|
||||
dst_ip = packet.dst_ip,
|
||||
src_port = packet.src_port,
|
||||
dst_port = packet.dst_port,
|
||||
protocol = packet.protocol,
|
||||
start_time = packet.timestamp,
|
||||
)
|
||||
self._flows[key] = flow
|
||||
|
||||
flow.packet_count += 1
|
||||
flow.total_bytes += len(packet.payload)
|
||||
flow.end_time = packet.timestamp
|
||||
|
||||
if packet.payload:
|
||||
flow.segments.append((packet.tcp_seq, packet.payload))
|
||||
|
||||
def get_flows(self) -> list[FlowStats]:
|
||||
"""
|
||||
Return all tracked flows
|
||||
"""
|
||||
return list(self._flows.values())
|
||||
|
||||
def get_flow(self, key: FlowKey) -> FlowStats | None:
|
||||
"""
|
||||
Get a specific flow by key
|
||||
"""
|
||||
return self._flows.get(key)
|
||||
|
||||
def reassemble_stream(self, key: FlowKey) -> bytes:
|
||||
"""
|
||||
Reassemble TCP payload ordered by sequence number
|
||||
"""
|
||||
flow = self._flows.get(key)
|
||||
if flow is None:
|
||||
return b""
|
||||
|
||||
sorted_segments = sorted(flow.segments, key = lambda s: s[0])
|
||||
|
||||
seen_offsets: set[int] = set()
|
||||
parts: list[bytes] = []
|
||||
for seq, data in sorted_segments:
|
||||
if seq not in seen_offsets:
|
||||
seen_offsets.add(seq)
|
||||
parts.append(data)
|
||||
|
||||
return b"".join(parts)
|
||||
|
||||
@property
|
||||
def flow_count(self) -> int:
|
||||
"""
|
||||
Return the number of tracked flows
|
||||
"""
|
||||
return len(self._flows)
|
||||
|
||||
|
||||
def make_flow_key(
|
||||
packet: PacketInfo,
|
||||
) -> FlowKey:
|
||||
"""
|
||||
Create a bidirectional flow key from a packet
|
||||
"""
|
||||
forward = (
|
||||
packet.src_ip,
|
||||
packet.dst_ip,
|
||||
packet.src_port,
|
||||
packet.dst_port,
|
||||
)
|
||||
reverse = (
|
||||
packet.dst_ip,
|
||||
packet.src_ip,
|
||||
packet.dst_port,
|
||||
packet.src_port,
|
||||
)
|
||||
return min(forward, reverse)
|
||||
|
|
@ -0,0 +1,115 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
pcap.py
|
||||
"""
|
||||
|
||||
|
||||
import socket
|
||||
from collections.abc import Iterator
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
import structlog
|
||||
|
||||
|
||||
log = structlog.get_logger()
|
||||
|
||||
TCP_PROTO: int = 6
|
||||
UDP_PROTO: int = 17
|
||||
|
||||
|
||||
@dataclass(frozen = True, slots = True)
|
||||
class PacketInfo:
|
||||
"""
|
||||
Parsed network packet with extracted metadata
|
||||
"""
|
||||
|
||||
timestamp: float
|
||||
src_ip: str
|
||||
dst_ip: str
|
||||
src_port: int
|
||||
dst_port: int
|
||||
protocol: str
|
||||
payload: bytes
|
||||
raw_length: int
|
||||
tcp_flags: int = 0
|
||||
tcp_seq: int = 0
|
||||
|
||||
|
||||
def read_pcap(
|
||||
path: Path,
|
||||
max_packets: int = 0,
|
||||
) -> Iterator[PacketInfo]:
|
||||
"""
|
||||
Read packets from a PCAP or PCAPNG file
|
||||
"""
|
||||
import dpkt
|
||||
|
||||
with open(path, "rb") as f:
|
||||
try:
|
||||
pcap = dpkt.pcap.Reader(f)
|
||||
except ValueError:
|
||||
f.seek(0)
|
||||
pcap = dpkt.pcapng.Reader(f)
|
||||
|
||||
count = 0
|
||||
for timestamp, buf in pcap:
|
||||
if max_packets > 0 and count >= max_packets:
|
||||
break
|
||||
|
||||
packet = _parse_ethernet(timestamp, buf)
|
||||
if packet is not None:
|
||||
yield packet
|
||||
count += 1
|
||||
|
||||
|
||||
def _parse_ethernet(
|
||||
timestamp: float,
|
||||
buf: bytes,
|
||||
) -> PacketInfo | None:
|
||||
"""
|
||||
Parse an Ethernet frame into a PacketInfo
|
||||
"""
|
||||
import dpkt
|
||||
|
||||
try:
|
||||
eth = dpkt.ethernet.Ethernet(buf)
|
||||
except (dpkt.NeedData, dpkt.UnpackError):
|
||||
return None
|
||||
|
||||
if not isinstance(eth.data, dpkt.ip.IP):
|
||||
return None
|
||||
|
||||
ip_pkt = eth.data
|
||||
src_ip = socket.inet_ntoa(ip_pkt.src)
|
||||
dst_ip = socket.inet_ntoa(ip_pkt.dst)
|
||||
|
||||
if isinstance(ip_pkt.data, dpkt.tcp.TCP):
|
||||
tcp = ip_pkt.data
|
||||
return PacketInfo(
|
||||
timestamp = timestamp,
|
||||
src_ip = src_ip,
|
||||
dst_ip = dst_ip,
|
||||
src_port = tcp.sport,
|
||||
dst_port = tcp.dport,
|
||||
protocol = "tcp",
|
||||
payload = bytes(tcp.data),
|
||||
raw_length = len(buf),
|
||||
tcp_flags = tcp.flags,
|
||||
tcp_seq = tcp.seq,
|
||||
)
|
||||
|
||||
if isinstance(ip_pkt.data, dpkt.udp.UDP):
|
||||
udp = ip_pkt.data
|
||||
return PacketInfo(
|
||||
timestamp = timestamp,
|
||||
src_ip = src_ip,
|
||||
dst_ip = dst_ip,
|
||||
src_port = udp.sport,
|
||||
dst_port = udp.dport,
|
||||
protocol = "udp",
|
||||
payload = bytes(udp.data),
|
||||
raw_length = len(buf),
|
||||
)
|
||||
|
||||
return None
|
||||
|
|
@ -0,0 +1,250 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
protocols.py
|
||||
"""
|
||||
|
||||
|
||||
import socket
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
import structlog
|
||||
|
||||
|
||||
log = structlog.get_logger()
|
||||
|
||||
HTTP_METHODS: frozenset[bytes] = frozenset(
|
||||
{
|
||||
b"GET",
|
||||
b"POST",
|
||||
b"PUT",
|
||||
b"DELETE",
|
||||
b"HEAD",
|
||||
b"OPTIONS",
|
||||
b"PATCH",
|
||||
}
|
||||
)
|
||||
|
||||
HTTP_RESPONSE_PREFIX: bytes = b"HTTP/"
|
||||
TLS_RECORD_PREFIX: bytes = b"\x16\x03"
|
||||
SSH_PREFIX: bytes = b"SSH-"
|
||||
SMTP_BANNER_PREFIX: bytes = b"220 "
|
||||
DNS_PORT: int = 53
|
||||
|
||||
DNS_QTYPES: dict[int,
|
||||
str] = {
|
||||
1: "A",
|
||||
2: "NS",
|
||||
5: "CNAME",
|
||||
6: "SOA",
|
||||
12: "PTR",
|
||||
15: "MX",
|
||||
16: "TXT",
|
||||
28: "AAAA",
|
||||
33: "SRV",
|
||||
255: "ANY",
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen = True, slots = True)
|
||||
class HttpMessage:
|
||||
"""
|
||||
Parsed HTTP request or response
|
||||
"""
|
||||
|
||||
method: str
|
||||
uri: str
|
||||
version: str
|
||||
headers: dict[str, str]
|
||||
body: str
|
||||
is_request: bool
|
||||
|
||||
|
||||
@dataclass(frozen = True, slots = True)
|
||||
class DnsQuery:
|
||||
"""
|
||||
A single DNS query entry
|
||||
"""
|
||||
|
||||
name: str
|
||||
query_type: str
|
||||
query_class: str
|
||||
|
||||
|
||||
@dataclass(frozen = True, slots = True)
|
||||
class DnsRecord:
|
||||
"""
|
||||
Parsed DNS message with queries and answers
|
||||
"""
|
||||
|
||||
queries: list[DnsQuery] = field(default_factory = list)
|
||||
answers: list[str] = field(default_factory = list)
|
||||
is_response: bool = False
|
||||
transaction_id: int = 0
|
||||
|
||||
|
||||
def parse_http(
|
||||
payload: bytes,
|
||||
) -> HttpMessage | None:
|
||||
"""
|
||||
Parse HTTP request or response from raw payload
|
||||
"""
|
||||
import dpkt
|
||||
|
||||
try:
|
||||
if _is_http_request(payload):
|
||||
req = dpkt.http.Request(payload)
|
||||
headers = dict(req.headers)
|
||||
body = _decode_body(req.body)
|
||||
return HttpMessage(
|
||||
method = req.method,
|
||||
uri = req.uri,
|
||||
version = req.version,
|
||||
headers = headers,
|
||||
body = body,
|
||||
is_request = True,
|
||||
)
|
||||
|
||||
if payload.startswith(HTTP_RESPONSE_PREFIX):
|
||||
resp = dpkt.http.Response(payload)
|
||||
headers = dict(resp.headers)
|
||||
body = _decode_body(resp.body)
|
||||
return HttpMessage(
|
||||
method = "",
|
||||
uri = "",
|
||||
version = resp.version,
|
||||
headers = headers,
|
||||
body = body,
|
||||
is_request = False,
|
||||
)
|
||||
except (dpkt.NeedData, dpkt.UnpackError):
|
||||
return None
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def parse_dns(
|
||||
payload: bytes,
|
||||
) -> DnsRecord | None:
|
||||
"""
|
||||
Parse DNS message from raw UDP payload
|
||||
"""
|
||||
import dpkt
|
||||
|
||||
try:
|
||||
dns = dpkt.dns.DNS(payload)
|
||||
except (dpkt.NeedData, dpkt.UnpackError):
|
||||
return None
|
||||
|
||||
queries: list[DnsQuery] = []
|
||||
for qd in dns.qd:
|
||||
qtype = DNS_QTYPES.get(qd.type, str(qd.type))
|
||||
queries.append(
|
||||
DnsQuery(
|
||||
name = qd.name,
|
||||
query_type = qtype,
|
||||
query_class = str(qd.cls),
|
||||
)
|
||||
)
|
||||
|
||||
answers: list[str] = []
|
||||
for an in dns.an:
|
||||
_parse_answer(an, answers)
|
||||
|
||||
return DnsRecord(
|
||||
queries = queries,
|
||||
answers = answers,
|
||||
is_response = bool(dns.qr),
|
||||
transaction_id = dns.id,
|
||||
)
|
||||
|
||||
|
||||
def identify_protocol(
|
||||
payload: bytes,
|
||||
) -> str:
|
||||
"""
|
||||
Identify application-layer protocol via DPI
|
||||
"""
|
||||
if not payload:
|
||||
return "unknown"
|
||||
|
||||
if _is_http_request(payload):
|
||||
return "http"
|
||||
|
||||
if payload.startswith(HTTP_RESPONSE_PREFIX):
|
||||
return "http"
|
||||
|
||||
if (len(payload) > 2 and payload[: 2] == TLS_RECORD_PREFIX):
|
||||
return "tls"
|
||||
|
||||
if payload.startswith(SSH_PREFIX):
|
||||
return "ssh"
|
||||
|
||||
if payload.startswith(SMTP_BANNER_PREFIX):
|
||||
return "smtp"
|
||||
|
||||
return "unknown"
|
||||
|
||||
|
||||
def _is_http_request(payload: bytes) -> bool:
|
||||
"""
|
||||
Check if payload starts with an HTTP method
|
||||
"""
|
||||
first_space = payload.find(b" ")
|
||||
if first_space < 3 or first_space > 7:
|
||||
return False
|
||||
return payload[: first_space] in HTTP_METHODS
|
||||
|
||||
|
||||
def _decode_body(body: bytes | str) -> str:
|
||||
"""
|
||||
Decode HTTP body bytes to string
|
||||
"""
|
||||
if isinstance(body, str):
|
||||
return body
|
||||
if not body:
|
||||
return ""
|
||||
try:
|
||||
return body.decode("utf-8", errors = "replace")
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
|
||||
def _parse_answer(
|
||||
an: object,
|
||||
answers: list[str],
|
||||
) -> None:
|
||||
"""
|
||||
Parse a single DNS answer record
|
||||
"""
|
||||
try:
|
||||
an_type = getattr(an, "type", 0)
|
||||
rdata = getattr(an, "rdata", b"")
|
||||
|
||||
if an_type == 1 and len(rdata) == 4:
|
||||
answers.append(socket.inet_ntoa(rdata))
|
||||
elif an_type == 16 and rdata:
|
||||
answers.append(_parse_txt_rdata(rdata))
|
||||
elif hasattr(an, "cname") and an.cname:
|
||||
answers.append(an.cname)
|
||||
elif hasattr(an, "name") and an.name:
|
||||
answers.append(an.name)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def _parse_txt_rdata(rdata: bytes) -> str:
|
||||
"""
|
||||
Parse TXT record rdata (length-prefixed strings)
|
||||
"""
|
||||
parts: list[str] = []
|
||||
i = 0
|
||||
while i < len(rdata):
|
||||
length = rdata[i]
|
||||
i += 1
|
||||
if i + length <= len(rdata):
|
||||
chunk = rdata[i : i + length]
|
||||
parts.append(chunk.decode("utf-8", errors = "replace"))
|
||||
i += length
|
||||
else:
|
||||
break
|
||||
return " ".join(parts)
|
||||
|
|
@ -0,0 +1,84 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
redaction.py
|
||||
"""
|
||||
|
||||
|
||||
from dlp_scanner.constants import RedactionStyle
|
||||
|
||||
|
||||
REDACTED_LABEL: str = "[REDACTED]"
|
||||
MASK_CHAR: str = "*"
|
||||
SNIPPET_CONTEXT_CHARS: int = 20
|
||||
|
||||
|
||||
def redact(
|
||||
text: str,
|
||||
start: int,
|
||||
end: int,
|
||||
style: RedactionStyle = "partial",
|
||||
) -> str:
|
||||
"""
|
||||
Redact matched text according to the chosen strategy
|
||||
"""
|
||||
matched = text[start : end]
|
||||
|
||||
if style == "none":
|
||||
return _build_snippet(text, start, end, matched)
|
||||
|
||||
if style == "full":
|
||||
return _build_snippet(text, start, end, REDACTED_LABEL)
|
||||
|
||||
redacted = _partial_redact(matched)
|
||||
return _build_snippet(text, start, end, redacted)
|
||||
|
||||
|
||||
def _partial_redact(value: str) -> str:
|
||||
"""
|
||||
Partially mask a value, keeping the last few chars visible
|
||||
"""
|
||||
stripped = value.replace("-", "").replace(" ", "")
|
||||
|
||||
if len(stripped) >= 9 and stripped.isdigit():
|
||||
return MASK_CHAR * (len(value) - 4) + value[-4 :]
|
||||
|
||||
if "@" in value:
|
||||
local, domain = value.rsplit("@", maxsplit = 1)
|
||||
masked_local = local[0] + MASK_CHAR * (len(local) - 1)
|
||||
return f"{masked_local}@{domain}"
|
||||
|
||||
if len(value) > 8:
|
||||
visible = max(4, len(value) // 4)
|
||||
return (MASK_CHAR * (len(value) - visible) + value[-visible :])
|
||||
|
||||
return MASK_CHAR * len(value)
|
||||
|
||||
|
||||
def _build_snippet(
|
||||
text: str,
|
||||
start: int,
|
||||
end: int,
|
||||
replacement: str,
|
||||
) -> str:
|
||||
"""
|
||||
Build a snippet with context around the redacted match
|
||||
"""
|
||||
context_start = max(0, start - SNIPPET_CONTEXT_CHARS)
|
||||
context_end = min(len(text), end + SNIPPET_CONTEXT_CHARS)
|
||||
|
||||
prefix = text[context_start : start]
|
||||
suffix = text[end : context_end]
|
||||
|
||||
prefix = prefix.replace("\n", " ").strip()
|
||||
suffix = suffix.replace("\n", " ").strip()
|
||||
|
||||
parts: list[str] = []
|
||||
if context_start > 0:
|
||||
parts.append("...")
|
||||
parts.append(prefix)
|
||||
parts.append(replacement)
|
||||
parts.append(suffix)
|
||||
if context_end < len(text):
|
||||
parts.append("...")
|
||||
|
||||
return "".join(parts)
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
__init__.py
|
||||
"""
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
base.py
|
||||
"""
|
||||
|
||||
|
||||
from typing import Protocol
|
||||
|
||||
from dlp_scanner.models import ScanResult
|
||||
|
||||
|
||||
class Reporter(Protocol):
|
||||
"""
|
||||
Protocol for all report output formats
|
||||
"""
|
||||
def generate(self, result: ScanResult) -> str:
|
||||
"""
|
||||
Generate report content as a string
|
||||
"""
|
||||
...
|
||||
|
|
@ -0,0 +1,162 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
console.py
|
||||
"""
|
||||
|
||||
|
||||
from rich.console import Console
|
||||
from rich.table import Table
|
||||
|
||||
from dlp_scanner.constants import SEVERITY_COLORS
|
||||
from dlp_scanner.models import ScanResult
|
||||
|
||||
|
||||
TRUNCATE_SNIPPET: int = 60
|
||||
|
||||
|
||||
class ConsoleReporter:
|
||||
"""
|
||||
Rich console output with severity-colored tables
|
||||
"""
|
||||
def __init__(
|
||||
self,
|
||||
console: Console | None = None,
|
||||
) -> None:
|
||||
self._console = console or Console()
|
||||
|
||||
def generate(self, result: ScanResult) -> str:
|
||||
"""
|
||||
Generate plain-text table for piping
|
||||
"""
|
||||
lines: list[str] = []
|
||||
lines.append(
|
||||
f"Scan {result.scan_id} | "
|
||||
f"{len(result.findings)} findings | "
|
||||
f"{result.targets_scanned} targets"
|
||||
)
|
||||
lines.append("")
|
||||
|
||||
for finding in result.findings:
|
||||
loc = finding.location.uri
|
||||
if finding.location.line is not None:
|
||||
loc += f":{finding.location.line}"
|
||||
if finding.location.table_name:
|
||||
loc += (f" [{finding.location.table_name}]")
|
||||
|
||||
snippet = finding.redacted_snippet
|
||||
if len(snippet) > TRUNCATE_SNIPPET:
|
||||
snippet = (snippet[: TRUNCATE_SNIPPET] + "...")
|
||||
|
||||
frameworks = ", ".join(finding.compliance_frameworks)
|
||||
|
||||
lines.append(
|
||||
f"[{finding.severity.upper()}] "
|
||||
f"{finding.rule_name} | "
|
||||
f"{loc} | "
|
||||
f"{finding.confidence:.0%} | "
|
||||
f"{snippet} | "
|
||||
f"{frameworks}"
|
||||
)
|
||||
|
||||
lines.append("")
|
||||
lines.append(_format_summary(result))
|
||||
return "\n".join(lines)
|
||||
|
||||
def display(self, result: ScanResult) -> None:
|
||||
"""
|
||||
Print Rich-formatted table to console
|
||||
"""
|
||||
self._console.print()
|
||||
|
||||
if not result.findings:
|
||||
self._console.print("[green]No findings detected.[/green]")
|
||||
_print_summary(self._console, result)
|
||||
return
|
||||
|
||||
table = Table(
|
||||
title = (
|
||||
f"DLP Scan Results "
|
||||
f"({len(result.findings)} findings)"
|
||||
),
|
||||
show_lines = True,
|
||||
)
|
||||
|
||||
table.add_column("Severity", width = 10, justify = "center")
|
||||
table.add_column("Rule", width = 25)
|
||||
table.add_column("Location", width = 30)
|
||||
table.add_column("Confidence", width = 10)
|
||||
table.add_column("Snippet", width = 40)
|
||||
table.add_column("Compliance", width = 20)
|
||||
|
||||
for finding in result.findings:
|
||||
color = SEVERITY_COLORS.get(finding.severity, "white")
|
||||
|
||||
loc = finding.location.uri
|
||||
if finding.location.line is not None:
|
||||
loc += f":{finding.location.line}"
|
||||
if finding.location.table_name:
|
||||
loc += (f"\n[{finding.location.table_name}]")
|
||||
|
||||
snippet = finding.redacted_snippet
|
||||
if len(snippet) > TRUNCATE_SNIPPET:
|
||||
snippet = (snippet[: TRUNCATE_SNIPPET] + "...")
|
||||
|
||||
frameworks = "\n".join(finding.compliance_frameworks)
|
||||
|
||||
table.add_row(
|
||||
f"[{color}]{finding.severity.upper()}"
|
||||
f"[/{color}]",
|
||||
finding.rule_name,
|
||||
loc,
|
||||
f"{finding.confidence:.0%}",
|
||||
snippet,
|
||||
frameworks,
|
||||
)
|
||||
|
||||
self._console.print(table)
|
||||
_print_summary(self._console, result)
|
||||
|
||||
if result.errors:
|
||||
self._console.print()
|
||||
self._console.print(
|
||||
f"[yellow]{len(result.errors)} "
|
||||
f"error(s) during scan[/yellow]"
|
||||
)
|
||||
|
||||
|
||||
def _format_summary(result: ScanResult) -> str:
|
||||
"""
|
||||
Format summary statistics as plain text
|
||||
"""
|
||||
by_sev = result.findings_by_severity
|
||||
parts: list[str] = []
|
||||
for sev in ("critical", "high", "medium", "low"):
|
||||
count = by_sev.get(sev, 0)
|
||||
if count > 0:
|
||||
parts.append(f"{sev}: {count}")
|
||||
|
||||
summary = " | ".join(parts) if parts else "clean"
|
||||
return (
|
||||
f"Summary: {summary} "
|
||||
f"({result.targets_scanned} targets scanned)"
|
||||
)
|
||||
|
||||
|
||||
def _print_summary(console: Console, result: ScanResult) -> None:
|
||||
"""
|
||||
Print formatted summary using Rich
|
||||
"""
|
||||
console.print()
|
||||
by_sev = result.findings_by_severity
|
||||
parts: list[str] = []
|
||||
for sev in ("critical", "high", "medium", "low"):
|
||||
count = by_sev.get(sev, 0)
|
||||
if count > 0:
|
||||
color = SEVERITY_COLORS.get(sev, "white")
|
||||
parts.append(f"[{color}]{sev}: {count}[/{color}]")
|
||||
|
||||
summary = (" | ".join(parts) if parts else "[green]clean")
|
||||
console.print(
|
||||
f"Summary: {summary} "
|
||||
f"({result.targets_scanned} targets)"
|
||||
)
|
||||
|
|
@ -0,0 +1,64 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
csv_report.py
|
||||
"""
|
||||
|
||||
|
||||
import csv
|
||||
import io
|
||||
|
||||
from dlp_scanner.models import ScanResult
|
||||
|
||||
|
||||
CSV_COLUMNS: list[str] = [
|
||||
"finding_id",
|
||||
"scan_date",
|
||||
"severity",
|
||||
"confidence",
|
||||
"rule_id",
|
||||
"rule_name",
|
||||
"source_type",
|
||||
"uri",
|
||||
"line",
|
||||
"column",
|
||||
"table_name",
|
||||
"redacted_snippet",
|
||||
"compliance_frameworks",
|
||||
"remediation",
|
||||
]
|
||||
|
||||
|
||||
class CsvReporter:
|
||||
"""
|
||||
CSV export for compliance team consumption
|
||||
"""
|
||||
def generate(self, result: ScanResult) -> str:
|
||||
"""
|
||||
Generate CSV report as a string
|
||||
"""
|
||||
output = io.StringIO()
|
||||
writer = csv.writer(output)
|
||||
writer.writerow(CSV_COLUMNS)
|
||||
|
||||
for finding in result.findings:
|
||||
frameworks = ";".join(finding.compliance_frameworks)
|
||||
writer.writerow(
|
||||
[
|
||||
finding.finding_id,
|
||||
finding.detected_at.isoformat(),
|
||||
finding.severity,
|
||||
f"{finding.confidence:.4f}",
|
||||
finding.rule_id,
|
||||
finding.rule_name,
|
||||
finding.location.source_type,
|
||||
finding.location.uri,
|
||||
finding.location.line or "",
|
||||
finding.location.column or "",
|
||||
finding.location.table_name or "",
|
||||
finding.redacted_snippet,
|
||||
frameworks,
|
||||
finding.remediation,
|
||||
]
|
||||
)
|
||||
|
||||
return output.getvalue()
|
||||
|
|
@ -0,0 +1,110 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
json_report.py
|
||||
"""
|
||||
|
||||
|
||||
from typing import Any
|
||||
|
||||
import orjson
|
||||
|
||||
from dlp_scanner.models import ScanResult
|
||||
|
||||
|
||||
class JsonReporter:
|
||||
"""
|
||||
Structured JSON report with metadata and summary
|
||||
"""
|
||||
def generate(self, result: ScanResult) -> str:
|
||||
"""
|
||||
Generate JSON report as a formatted string
|
||||
"""
|
||||
report = _build_report(result)
|
||||
return orjson.dumps(
|
||||
report,
|
||||
option = (orjson.OPT_INDENT_2
|
||||
| orjson.OPT_NON_STR_KEYS),
|
||||
).decode("utf-8")
|
||||
|
||||
|
||||
def _build_report(
|
||||
result: ScanResult,
|
||||
) -> dict[str,
|
||||
Any]:
|
||||
"""
|
||||
Build the complete report structure
|
||||
"""
|
||||
return {
|
||||
"scan_metadata": _build_metadata(result),
|
||||
"findings": [_serialize_finding(f) for f in result.findings],
|
||||
"summary": _build_summary(result),
|
||||
}
|
||||
|
||||
|
||||
def _build_metadata(
|
||||
result: ScanResult,
|
||||
) -> dict[str,
|
||||
Any]:
|
||||
"""
|
||||
Build scan metadata section
|
||||
"""
|
||||
return {
|
||||
"scan_id":
|
||||
result.scan_id,
|
||||
"tool_version":
|
||||
result.tool_version,
|
||||
"scan_started_at": (result.scan_started_at.isoformat()),
|
||||
"scan_completed_at": (
|
||||
result.scan_completed_at.isoformat()
|
||||
if result.scan_completed_at else None
|
||||
),
|
||||
"targets_scanned":
|
||||
result.targets_scanned,
|
||||
"total_findings":
|
||||
len(result.findings),
|
||||
"errors":
|
||||
result.errors,
|
||||
}
|
||||
|
||||
|
||||
def _serialize_finding(
|
||||
finding: Any,
|
||||
) -> dict[str,
|
||||
Any]:
|
||||
"""
|
||||
Serialize a single finding to dict
|
||||
"""
|
||||
return {
|
||||
"finding_id": finding.finding_id,
|
||||
"rule_id": finding.rule_id,
|
||||
"rule_name": finding.rule_name,
|
||||
"severity": finding.severity,
|
||||
"confidence": round(finding.confidence,
|
||||
4),
|
||||
"location": {
|
||||
"source_type": (finding.location.source_type),
|
||||
"uri": finding.location.uri,
|
||||
"line": finding.location.line,
|
||||
"column": finding.location.column,
|
||||
"table_name": (finding.location.table_name),
|
||||
"column_name": (finding.location.column_name),
|
||||
},
|
||||
"redacted_snippet": (finding.redacted_snippet),
|
||||
"compliance_frameworks": (finding.compliance_frameworks),
|
||||
"remediation": finding.remediation,
|
||||
"detected_at": (finding.detected_at.isoformat()),
|
||||
}
|
||||
|
||||
|
||||
def _build_summary(
|
||||
result: ScanResult,
|
||||
) -> dict[str,
|
||||
Any]:
|
||||
"""
|
||||
Build summary statistics section
|
||||
"""
|
||||
return {
|
||||
"by_severity": result.findings_by_severity,
|
||||
"by_rule": result.findings_by_rule,
|
||||
"by_framework": result.findings_by_framework,
|
||||
}
|
||||
|
|
@ -0,0 +1,171 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
sarif.py
|
||||
"""
|
||||
|
||||
|
||||
from typing import Any
|
||||
|
||||
import orjson
|
||||
|
||||
from dlp_scanner.constants import SARIF_SEVERITY_MAP
|
||||
from dlp_scanner.models import Finding, ScanResult
|
||||
|
||||
|
||||
SARIF_SCHEMA: str = (
|
||||
"https://raw.githubusercontent.com/"
|
||||
"oasis-tcs/sarif-spec/main/sarif-2.1/"
|
||||
"schema/sarif-schema-2.1.0.json"
|
||||
)
|
||||
SARIF_VERSION: str = "2.1.0"
|
||||
TOOL_NAME: str = "dlp-scanner"
|
||||
|
||||
|
||||
class SarifReporter:
|
||||
"""
|
||||
SARIF 2.1.0 output for CI/CD integration
|
||||
"""
|
||||
def generate(self, result: ScanResult) -> str:
|
||||
"""
|
||||
Generate SARIF 2.1.0 report as JSON string
|
||||
"""
|
||||
sarif = _build_sarif(result)
|
||||
return orjson.dumps(
|
||||
sarif,
|
||||
option = (orjson.OPT_INDENT_2
|
||||
| orjson.OPT_NON_STR_KEYS),
|
||||
).decode("utf-8")
|
||||
|
||||
|
||||
def _build_sarif(
|
||||
result: ScanResult,
|
||||
) -> dict[str,
|
||||
Any]:
|
||||
"""
|
||||
Build complete SARIF document
|
||||
"""
|
||||
rules = _collect_rules(result.findings)
|
||||
results = [_build_result(f, rules) for f in result.findings]
|
||||
|
||||
return {
|
||||
"$schema":
|
||||
SARIF_SCHEMA,
|
||||
"version":
|
||||
SARIF_VERSION,
|
||||
"runs": [
|
||||
{
|
||||
"tool": {
|
||||
"driver": {
|
||||
"name": TOOL_NAME,
|
||||
"version": (result.tool_version),
|
||||
"rules": list(rules.values()),
|
||||
}
|
||||
},
|
||||
"results": results,
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _collect_rules(
|
||||
findings: list[Finding],
|
||||
) -> dict[str,
|
||||
dict[str,
|
||||
Any]]:
|
||||
"""
|
||||
Collect unique rules from findings
|
||||
"""
|
||||
rules: dict[str, dict[str, Any]] = {}
|
||||
|
||||
for finding in findings:
|
||||
if finding.rule_id in rules:
|
||||
continue
|
||||
|
||||
rules[finding.rule_id] = {
|
||||
"id": finding.rule_id,
|
||||
"name": finding.rule_name,
|
||||
"shortDescription": {
|
||||
"text": finding.rule_name,
|
||||
},
|
||||
"properties": {
|
||||
"compliance_frameworks": (finding.compliance_frameworks),
|
||||
},
|
||||
}
|
||||
|
||||
return rules
|
||||
|
||||
|
||||
def _build_result(
|
||||
finding: Finding,
|
||||
rules: dict[str,
|
||||
dict[str,
|
||||
Any]],
|
||||
) -> dict[str,
|
||||
Any]:
|
||||
"""
|
||||
Build a single SARIF result entry
|
||||
"""
|
||||
level = SARIF_SEVERITY_MAP.get(finding.severity, "note")
|
||||
|
||||
location = _build_location(finding)
|
||||
|
||||
return {
|
||||
"ruleId": finding.rule_id,
|
||||
"ruleIndex": list(rules.keys()).index(finding.rule_id),
|
||||
"level": level,
|
||||
"message": {
|
||||
"text": (
|
||||
f"{finding.rule_name} detected "
|
||||
f"with {finding.confidence:.0%} "
|
||||
f"confidence"
|
||||
),
|
||||
},
|
||||
"locations": [location],
|
||||
"properties": {
|
||||
"confidence": round(finding.confidence,
|
||||
4),
|
||||
"redactedSnippet": (finding.redacted_snippet),
|
||||
"complianceFrameworks": (finding.compliance_frameworks),
|
||||
"remediation": finding.remediation,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _build_location(
|
||||
finding: Finding,
|
||||
) -> dict[str,
|
||||
Any]:
|
||||
"""
|
||||
Build SARIF location from finding
|
||||
"""
|
||||
loc = finding.location
|
||||
|
||||
physical: dict[str,
|
||||
Any] = {
|
||||
"artifactLocation": {
|
||||
"uri": loc.uri
|
||||
},
|
||||
}
|
||||
|
||||
region: dict[str, Any] = {}
|
||||
if loc.line is not None:
|
||||
region["startLine"] = loc.line
|
||||
if loc.column is not None:
|
||||
region["startColumn"] = loc.column
|
||||
if region:
|
||||
physical["region"] = region
|
||||
|
||||
result: dict[str,
|
||||
Any] = {
|
||||
"physicalLocation": physical,
|
||||
}
|
||||
|
||||
if loc.table_name:
|
||||
result["logicalLocations"] = [
|
||||
{
|
||||
"name": loc.table_name,
|
||||
"kind": "table",
|
||||
}
|
||||
]
|
||||
|
||||
return result
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
__init__.py
|
||||
"""
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
base.py
|
||||
"""
|
||||
|
||||
|
||||
from typing import Protocol
|
||||
|
||||
from dlp_scanner.models import ScanResult
|
||||
|
||||
|
||||
class Scanner(Protocol):
|
||||
"""
|
||||
Protocol for all scan strategies
|
||||
"""
|
||||
def scan(self, target: str) -> ScanResult:
|
||||
"""
|
||||
Scan the target and return aggregated results
|
||||
"""
|
||||
...
|
||||
|
|
@ -0,0 +1,530 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
db_scanner.py
|
||||
"""
|
||||
|
||||
|
||||
import asyncio
|
||||
from datetime import datetime, UTC
|
||||
from typing import Any
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import structlog
|
||||
|
||||
from dlp_scanner.config import ScanConfig
|
||||
from dlp_scanner.constants import (
|
||||
TEXT_DB_COLUMN_TYPES_MYSQL,
|
||||
TEXT_DB_COLUMN_TYPES_PG,
|
||||
)
|
||||
from dlp_scanner.detectors.base import DetectorMatch
|
||||
from dlp_scanner.detectors.registry import DetectorRegistry
|
||||
from dlp_scanner.models import (
|
||||
Location,
|
||||
ScanResult,
|
||||
)
|
||||
from dlp_scanner.scoring import match_to_finding
|
||||
|
||||
|
||||
log = structlog.get_logger()
|
||||
|
||||
POSTGRES_SCHEMES: frozenset[str] = frozenset({
|
||||
"postgresql",
|
||||
"postgres",
|
||||
})
|
||||
MYSQL_SCHEMES: frozenset[str] = frozenset({
|
||||
"mysql",
|
||||
"mysql+aiomysql",
|
||||
})
|
||||
MONGODB_SCHEMES: frozenset[str] = frozenset({
|
||||
"mongodb",
|
||||
"mongodb+srv",
|
||||
})
|
||||
SQLITE_SCHEMES: frozenset[str] = frozenset({
|
||||
"sqlite",
|
||||
})
|
||||
|
||||
|
||||
class DatabaseScanner:
|
||||
"""
|
||||
Scans database tables for sensitive data in text columns
|
||||
"""
|
||||
def __init__(
|
||||
self,
|
||||
config: ScanConfig,
|
||||
registry: DetectorRegistry,
|
||||
) -> None:
|
||||
self._db_config = config.database
|
||||
self._detection_config = config.detection
|
||||
self._redaction_style = config.output.redaction_style
|
||||
self._registry = registry
|
||||
|
||||
def scan(self, target: str) -> ScanResult:
|
||||
"""
|
||||
Scan a database identified by connection URI
|
||||
"""
|
||||
return asyncio.run(self._scan_async(target))
|
||||
|
||||
async def _scan_async(
|
||||
self,
|
||||
connection_uri: str,
|
||||
) -> ScanResult:
|
||||
"""
|
||||
Dispatch to the appropriate database scanner
|
||||
"""
|
||||
result = ScanResult()
|
||||
parsed = urlparse(connection_uri)
|
||||
scheme = parsed.scheme.lower()
|
||||
|
||||
try:
|
||||
if scheme in POSTGRES_SCHEMES:
|
||||
await self._scan_postgres(connection_uri, result)
|
||||
elif scheme in MYSQL_SCHEMES:
|
||||
await self._scan_mysql(connection_uri, result)
|
||||
elif scheme in MONGODB_SCHEMES:
|
||||
await self._scan_mongodb(connection_uri, result)
|
||||
elif scheme in SQLITE_SCHEMES:
|
||||
await self._scan_sqlite(connection_uri, result)
|
||||
else:
|
||||
result.errors.append(
|
||||
f"Unsupported database scheme: "
|
||||
f"{scheme}"
|
||||
)
|
||||
except Exception as exc:
|
||||
log.warning(
|
||||
"database_scan_failed",
|
||||
scheme = scheme,
|
||||
error = str(exc),
|
||||
)
|
||||
result.errors.append(f"Database scan failed: {exc}")
|
||||
|
||||
result.scan_completed_at = datetime.now(UTC)
|
||||
return result
|
||||
|
||||
async def _scan_postgres(
|
||||
self,
|
||||
uri: str,
|
||||
result: ScanResult,
|
||||
) -> None:
|
||||
"""
|
||||
Scan PostgreSQL using asyncpg with TABLESAMPLE
|
||||
"""
|
||||
import asyncpg
|
||||
|
||||
conn = await asyncpg.connect(
|
||||
uri,
|
||||
timeout = self._db_config.timeout_seconds,
|
||||
)
|
||||
|
||||
try:
|
||||
tables = await self._get_pg_tables(conn)
|
||||
tables = self._filter_tables(tables)
|
||||
|
||||
for table_name in tables:
|
||||
text_cols = (
|
||||
await self._get_pg_text_columns(conn,
|
||||
table_name)
|
||||
)
|
||||
if not text_cols:
|
||||
continue
|
||||
|
||||
col_list = ", ".join(f'"{c}"' for c in text_cols)
|
||||
query = (
|
||||
f"SELECT {col_list} "
|
||||
f'FROM "{table_name}" '
|
||||
f"TABLESAMPLE BERNOULLI("
|
||||
f"{self._db_config.sample_percentage}"
|
||||
f") LIMIT "
|
||||
f"{self._db_config.max_rows_per_table}"
|
||||
)
|
||||
|
||||
rows = await conn.fetch(query)
|
||||
self._process_record_rows(
|
||||
rows,
|
||||
text_cols,
|
||||
table_name,
|
||||
uri,
|
||||
result,
|
||||
)
|
||||
result.targets_scanned += 1
|
||||
finally:
|
||||
await conn.close()
|
||||
|
||||
async def _get_pg_tables(
|
||||
self,
|
||||
conn: Any,
|
||||
) -> list[str]:
|
||||
"""
|
||||
List user tables in PostgreSQL
|
||||
"""
|
||||
rows = await conn.fetch(
|
||||
"SELECT table_name "
|
||||
"FROM information_schema.tables "
|
||||
"WHERE table_schema = 'public' "
|
||||
"AND table_type = 'BASE TABLE'"
|
||||
)
|
||||
return [r["table_name"] for r in rows]
|
||||
|
||||
async def _get_pg_text_columns(
|
||||
self,
|
||||
conn: Any,
|
||||
table_name: str,
|
||||
) -> list[str]:
|
||||
"""
|
||||
Find text-type columns in a PostgreSQL table
|
||||
"""
|
||||
rows = await conn.fetch(
|
||||
"SELECT column_name "
|
||||
"FROM information_schema.columns "
|
||||
"WHERE table_name = $1 "
|
||||
"AND data_type = ANY($2::text[])",
|
||||
table_name,
|
||||
list(TEXT_DB_COLUMN_TYPES_PG),
|
||||
)
|
||||
return [r["column_name"] for r in rows]
|
||||
|
||||
async def _scan_mysql(
|
||||
self,
|
||||
uri: str,
|
||||
result: ScanResult,
|
||||
) -> None:
|
||||
"""
|
||||
Scan MySQL using aiomysql with random sampling
|
||||
"""
|
||||
import aiomysql
|
||||
|
||||
parsed = urlparse(uri)
|
||||
conn = await aiomysql.connect(
|
||||
host = parsed.hostname or "localhost",
|
||||
port = parsed.port or 3306,
|
||||
user = parsed.username or "root",
|
||||
password = parsed.password or "",
|
||||
db = parsed.path.lstrip("/"),
|
||||
connect_timeout = (self._db_config.timeout_seconds),
|
||||
)
|
||||
|
||||
try:
|
||||
async with conn.cursor(aiomysql.DictCursor) as cur:
|
||||
await cur.execute(
|
||||
"SELECT table_name "
|
||||
"FROM information_schema.tables "
|
||||
"WHERE table_schema = DATABASE() "
|
||||
"AND table_type = 'BASE TABLE'"
|
||||
)
|
||||
raw_tables = await cur.fetchall()
|
||||
tables = [r["TABLE_NAME"] for r in raw_tables]
|
||||
tables = self._filter_tables(tables)
|
||||
|
||||
for table_name in tables:
|
||||
text_cols = (
|
||||
await self._get_mysql_text_cols(cur,
|
||||
table_name)
|
||||
)
|
||||
if not text_cols:
|
||||
continue
|
||||
|
||||
col_list = ", ".join(f"`{c}`" for c in text_cols)
|
||||
limit = (self._db_config.max_rows_per_table)
|
||||
await cur.execute(
|
||||
f"SELECT {col_list} "
|
||||
f"FROM `{table_name}` "
|
||||
f"ORDER BY RAND() "
|
||||
f"LIMIT {limit}"
|
||||
)
|
||||
rows = await cur.fetchall()
|
||||
self._process_dict_rows(
|
||||
rows,
|
||||
text_cols,
|
||||
table_name,
|
||||
uri,
|
||||
result,
|
||||
)
|
||||
result.targets_scanned += 1
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
async def _get_mysql_text_cols(
|
||||
self,
|
||||
cursor: Any,
|
||||
table_name: str,
|
||||
) -> list[str]:
|
||||
"""
|
||||
Find text-type columns in a MySQL table
|
||||
"""
|
||||
placeholders = ",".join(["%s"] * len(TEXT_DB_COLUMN_TYPES_MYSQL))
|
||||
await cursor.execute(
|
||||
"SELECT column_name "
|
||||
"FROM information_schema.columns "
|
||||
"WHERE table_name = %s "
|
||||
"AND table_schema = DATABASE() "
|
||||
f"AND data_type IN ({placeholders})",
|
||||
(table_name,
|
||||
*TEXT_DB_COLUMN_TYPES_MYSQL),
|
||||
)
|
||||
rows = await cursor.fetchall()
|
||||
return [r["COLUMN_NAME"] for r in rows]
|
||||
|
||||
async def _scan_mongodb(
|
||||
self,
|
||||
uri: str,
|
||||
result: ScanResult,
|
||||
) -> None:
|
||||
"""
|
||||
Scan MongoDB collections using pymongo async
|
||||
"""
|
||||
from pymongo import AsyncMongoClient
|
||||
|
||||
parsed = urlparse(uri)
|
||||
db_name = parsed.path.lstrip("/").split("?")[0]
|
||||
|
||||
if not db_name:
|
||||
result.errors.append("MongoDB URI must include database name")
|
||||
return
|
||||
|
||||
client: AsyncMongoClient[dict[str, Any]] = (AsyncMongoClient(uri))
|
||||
|
||||
try:
|
||||
db = client[db_name]
|
||||
collections = (await db.list_collection_names())
|
||||
collections = self._filter_tables(collections)
|
||||
|
||||
for coll_name in collections:
|
||||
coll = db[coll_name]
|
||||
sample_size = (self._db_config.max_rows_per_table)
|
||||
cursor = coll.aggregate(
|
||||
[{
|
||||
"$sample": {
|
||||
"size": sample_size
|
||||
}
|
||||
}]
|
||||
)
|
||||
|
||||
async for doc in cursor:
|
||||
text_parts: list[str] = []
|
||||
_extract_mongo_strings(doc, text_parts)
|
||||
if not text_parts:
|
||||
continue
|
||||
|
||||
combined = "\n".join(text_parts)
|
||||
matches = self._registry.detect(combined)
|
||||
self._append_findings(
|
||||
matches,
|
||||
combined,
|
||||
table_name = coll_name,
|
||||
uri = uri,
|
||||
result = result,
|
||||
)
|
||||
|
||||
result.targets_scanned += 1
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
async def _scan_sqlite(
|
||||
self,
|
||||
uri: str,
|
||||
result: ScanResult,
|
||||
) -> None:
|
||||
"""
|
||||
Scan SQLite database using aiosqlite
|
||||
"""
|
||||
import aiosqlite
|
||||
|
||||
parsed = urlparse(uri)
|
||||
db_path = parsed.path
|
||||
while db_path.startswith("//"):
|
||||
db_path = db_path[1 :]
|
||||
|
||||
async with aiosqlite.connect(db_path) as db:
|
||||
cursor = await db.execute(
|
||||
"SELECT name FROM sqlite_master "
|
||||
"WHERE type = 'table' "
|
||||
"AND name NOT LIKE 'sqlite_%'"
|
||||
)
|
||||
rows = await cursor.fetchall()
|
||||
tables = [r[0] for r in rows]
|
||||
tables = self._filter_tables(tables)
|
||||
|
||||
for table_name in tables:
|
||||
text_cols = (
|
||||
await self._get_sqlite_text_cols(db,
|
||||
table_name)
|
||||
)
|
||||
if not text_cols:
|
||||
continue
|
||||
|
||||
col_list = ", ".join(f'"{c}"' for c in text_cols)
|
||||
limit = (self._db_config.max_rows_per_table)
|
||||
cursor = await db.execute(
|
||||
f"SELECT {col_list} "
|
||||
f'FROM "{table_name}" '
|
||||
f"ORDER BY RANDOM() "
|
||||
f"LIMIT {limit}"
|
||||
)
|
||||
fetched = await cursor.fetchall()
|
||||
for row in fetched:
|
||||
for idx, col_name in enumerate(text_cols):
|
||||
val = row[idx]
|
||||
if val is None:
|
||||
continue
|
||||
text = str(val)
|
||||
if not text.strip():
|
||||
continue
|
||||
matches = self._registry.detect(text)
|
||||
self._append_findings(
|
||||
matches,
|
||||
text,
|
||||
table_name = table_name,
|
||||
column_name = col_name,
|
||||
uri = uri,
|
||||
result = result,
|
||||
)
|
||||
result.targets_scanned += 1
|
||||
|
||||
async def _get_sqlite_text_cols(
|
||||
self,
|
||||
db: Any,
|
||||
table_name: str,
|
||||
) -> list[str]:
|
||||
"""
|
||||
Find text-type columns in a SQLite table
|
||||
"""
|
||||
cursor = await db.execute(f'PRAGMA table_info("{table_name}")')
|
||||
rows = await cursor.fetchall()
|
||||
text_types = frozenset({"text", "varchar", "char", "clob"})
|
||||
return [
|
||||
r[1]
|
||||
for r in rows
|
||||
if r[2].lower() in text_types or "text" in r[2].lower()
|
||||
]
|
||||
|
||||
def _filter_tables(
|
||||
self,
|
||||
tables: list[str],
|
||||
) -> list[str]:
|
||||
"""
|
||||
Apply include/exclude table filters
|
||||
"""
|
||||
include = self._db_config.include_tables
|
||||
exclude = frozenset(self._db_config.exclude_tables)
|
||||
|
||||
if include:
|
||||
include_set = frozenset(include)
|
||||
tables = [t for t in tables if t in include_set]
|
||||
|
||||
return [t for t in tables if t not in exclude]
|
||||
|
||||
def _process_record_rows(
|
||||
self,
|
||||
rows: list[Any],
|
||||
columns: list[str],
|
||||
table_name: str,
|
||||
uri: str,
|
||||
result: ScanResult,
|
||||
) -> None:
|
||||
"""
|
||||
Process asyncpg Record rows through detection
|
||||
"""
|
||||
for row in rows:
|
||||
for col_name in columns:
|
||||
val = row[col_name]
|
||||
if val is None:
|
||||
continue
|
||||
text = str(val)
|
||||
if not text.strip():
|
||||
continue
|
||||
matches = self._registry.detect(text)
|
||||
self._append_findings(
|
||||
matches,
|
||||
text,
|
||||
table_name = table_name,
|
||||
column_name = col_name,
|
||||
uri = uri,
|
||||
result = result,
|
||||
)
|
||||
|
||||
def _process_dict_rows(
|
||||
self,
|
||||
rows: list[dict[str,
|
||||
Any]],
|
||||
columns: list[str],
|
||||
table_name: str,
|
||||
uri: str,
|
||||
result: ScanResult,
|
||||
) -> None:
|
||||
"""
|
||||
Process dictionary rows through detection
|
||||
"""
|
||||
for row in rows:
|
||||
for col_name in columns:
|
||||
val = row.get(col_name)
|
||||
if val is None:
|
||||
continue
|
||||
text = str(val)
|
||||
if not text.strip():
|
||||
continue
|
||||
matches = self._registry.detect(text)
|
||||
self._append_findings(
|
||||
matches,
|
||||
text,
|
||||
table_name = table_name,
|
||||
column_name = col_name,
|
||||
uri = uri,
|
||||
result = result,
|
||||
)
|
||||
|
||||
def _append_findings(
|
||||
self,
|
||||
matches: list[DetectorMatch],
|
||||
text: str,
|
||||
table_name: str,
|
||||
uri: str,
|
||||
result: ScanResult,
|
||||
column_name: str = "",
|
||||
) -> None:
|
||||
"""
|
||||
Convert detector matches to findings and append
|
||||
"""
|
||||
min_confidence = (self._detection_config.min_confidence)
|
||||
|
||||
location = Location(
|
||||
source_type = "database",
|
||||
uri = uri,
|
||||
table_name = table_name,
|
||||
column_name = column_name or None,
|
||||
)
|
||||
|
||||
for match in matches:
|
||||
if match.score < min_confidence:
|
||||
continue
|
||||
|
||||
finding = match_to_finding(
|
||||
match,
|
||||
text,
|
||||
location,
|
||||
self._redaction_style,
|
||||
)
|
||||
result.findings.append(finding)
|
||||
|
||||
|
||||
def _extract_mongo_strings(
|
||||
doc: dict[str,
|
||||
Any],
|
||||
parts: list[str],
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
"""
|
||||
Recursively extract string values from a MongoDB document
|
||||
"""
|
||||
for key, val in doc.items():
|
||||
if key == "_id":
|
||||
continue
|
||||
key_path = (f"{prefix}.{key}" if prefix else key)
|
||||
if isinstance(val, str) and val.strip():
|
||||
parts.append(f"{key_path}: {val}")
|
||||
elif isinstance(val, dict):
|
||||
_extract_mongo_strings(val, parts, key_path)
|
||||
elif isinstance(val, list):
|
||||
for item in val:
|
||||
if (isinstance(item, str) and item.strip()):
|
||||
parts.append(f"{key_path}: {item}")
|
||||
elif isinstance(item, dict):
|
||||
_extract_mongo_strings(item, parts, key_path)
|
||||
|
|
@ -0,0 +1,224 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
file_scanner.py
|
||||
"""
|
||||
|
||||
|
||||
import fnmatch
|
||||
from datetime import datetime, UTC
|
||||
from pathlib import Path
|
||||
|
||||
import structlog
|
||||
|
||||
from dlp_scanner.config import ScanConfig
|
||||
from dlp_scanner.detectors.registry import DetectorRegistry
|
||||
from dlp_scanner.extractors.archive import ArchiveExtractor
|
||||
from dlp_scanner.extractors.base import Extractor
|
||||
from dlp_scanner.extractors.email import (
|
||||
EmlExtractor,
|
||||
MsgExtractor,
|
||||
)
|
||||
from dlp_scanner.extractors.office import (
|
||||
DocxExtractor,
|
||||
XlsExtractor,
|
||||
XlsxExtractor,
|
||||
)
|
||||
from dlp_scanner.extractors.pdf import PDFExtractor
|
||||
from dlp_scanner.extractors.plaintext import (
|
||||
PlaintextExtractor,
|
||||
)
|
||||
from dlp_scanner.extractors.structured import (
|
||||
AvroExtractor,
|
||||
CsvExtractor,
|
||||
JsonExtractor,
|
||||
ParquetExtractor,
|
||||
XmlExtractor,
|
||||
YamlExtractor,
|
||||
)
|
||||
from dlp_scanner.models import (
|
||||
ScanResult,
|
||||
TextChunk,
|
||||
)
|
||||
from dlp_scanner.scoring import match_to_finding
|
||||
|
||||
|
||||
log = structlog.get_logger()
|
||||
|
||||
MB_BYTES: int = 1024 * 1024
|
||||
|
||||
|
||||
class FileScanner:
|
||||
"""
|
||||
Scans files in a directory tree for sensitive data
|
||||
"""
|
||||
def __init__(
|
||||
self,
|
||||
config: ScanConfig,
|
||||
registry: DetectorRegistry,
|
||||
) -> None:
|
||||
self._file_config = config.file
|
||||
self._detection_config = config.detection
|
||||
self._redaction_style = config.output.redaction_style
|
||||
self._registry = registry
|
||||
self._extension_map = _build_extension_map()
|
||||
self._allowed_extensions = frozenset(
|
||||
self._file_config.include_extensions
|
||||
)
|
||||
|
||||
def scan(self, target: str) -> ScanResult:
|
||||
"""
|
||||
Walk a directory and scan all matching files
|
||||
"""
|
||||
result = ScanResult()
|
||||
target_path = Path(target)
|
||||
|
||||
if target_path.is_file():
|
||||
self._scan_file(target_path, result)
|
||||
result.targets_scanned = 1
|
||||
elif target_path.is_dir():
|
||||
self._scan_directory(target_path, result)
|
||||
else:
|
||||
result.errors.append(f"Target not found: {target}")
|
||||
|
||||
result.scan_completed_at = datetime.now(UTC)
|
||||
return result
|
||||
|
||||
def _scan_directory(
|
||||
self,
|
||||
directory: Path,
|
||||
result: ScanResult,
|
||||
) -> None:
|
||||
"""
|
||||
Recursively walk a directory and scan matching files
|
||||
"""
|
||||
max_bytes = (self._file_config.max_file_size_mb * MB_BYTES)
|
||||
iterator = (
|
||||
directory.rglob("*")
|
||||
if self._file_config.recursive else directory.glob("*")
|
||||
)
|
||||
|
||||
for path in iterator:
|
||||
if not path.is_file():
|
||||
continue
|
||||
|
||||
if self._is_excluded(path, directory):
|
||||
continue
|
||||
|
||||
suffix = _get_full_suffix(path)
|
||||
if suffix not in self._allowed_extensions:
|
||||
continue
|
||||
|
||||
try:
|
||||
file_size = path.stat().st_size
|
||||
except OSError:
|
||||
continue
|
||||
|
||||
if file_size > max_bytes:
|
||||
log.debug(
|
||||
"file_skipped_too_large",
|
||||
path = str(path),
|
||||
size = file_size,
|
||||
)
|
||||
continue
|
||||
|
||||
if file_size == 0:
|
||||
continue
|
||||
|
||||
self._scan_file(path, result)
|
||||
result.targets_scanned += 1
|
||||
|
||||
def _scan_file(
|
||||
self,
|
||||
path: Path,
|
||||
result: ScanResult,
|
||||
) -> None:
|
||||
"""
|
||||
Extract text from a single file and run detection
|
||||
"""
|
||||
suffix = _get_full_suffix(path)
|
||||
extractor = self._extension_map.get(suffix)
|
||||
|
||||
if extractor is None:
|
||||
return
|
||||
|
||||
try:
|
||||
chunks = extractor.extract(str(path))
|
||||
except Exception:
|
||||
log.warning("extraction_failed", path = str(path))
|
||||
result.errors.append(f"Extraction failed: {path}")
|
||||
return
|
||||
|
||||
min_confidence = (self._detection_config.min_confidence)
|
||||
|
||||
for chunk in chunks:
|
||||
matches = self._registry.detect(chunk.text)
|
||||
for match in matches:
|
||||
if match.score < min_confidence:
|
||||
continue
|
||||
|
||||
finding = match_to_finding(
|
||||
match,
|
||||
chunk.text,
|
||||
chunk.location,
|
||||
self._redaction_style,
|
||||
)
|
||||
result.findings.append(finding)
|
||||
|
||||
def _is_excluded(
|
||||
self,
|
||||
path: Path,
|
||||
base: Path,
|
||||
) -> bool:
|
||||
"""
|
||||
Check if a path matches any exclude pattern
|
||||
"""
|
||||
relative = str(path.relative_to(base))
|
||||
for pattern in self._file_config.exclude_patterns:
|
||||
if fnmatch.fnmatch(relative, pattern):
|
||||
return True
|
||||
if fnmatch.fnmatch(path.name, pattern):
|
||||
return True
|
||||
if any(fnmatch.fnmatch(part, pattern) for part in path.parts):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _build_extension_map() -> dict[str, Extractor]:
|
||||
"""
|
||||
Build a mapping from file extension to extractor instance
|
||||
"""
|
||||
extractors: list[Extractor] = [
|
||||
PlaintextExtractor(),
|
||||
PDFExtractor(),
|
||||
DocxExtractor(),
|
||||
XlsxExtractor(),
|
||||
XlsExtractor(),
|
||||
CsvExtractor(),
|
||||
JsonExtractor(),
|
||||
XmlExtractor(),
|
||||
YamlExtractor(),
|
||||
ParquetExtractor(),
|
||||
AvroExtractor(),
|
||||
ArchiveExtractor(),
|
||||
EmlExtractor(),
|
||||
MsgExtractor(),
|
||||
]
|
||||
|
||||
ext_map: dict[str, Extractor] = {}
|
||||
for extractor in extractors:
|
||||
for ext in extractor.supported_extensions:
|
||||
ext_map[ext] = extractor
|
||||
|
||||
return ext_map
|
||||
|
||||
|
||||
def _get_full_suffix(path: Path) -> str:
|
||||
"""
|
||||
Get full suffix including compound extensions
|
||||
"""
|
||||
name = path.name
|
||||
if name.endswith(".tar.gz"):
|
||||
return ".tar.gz"
|
||||
if name.endswith(".tar.bz2"):
|
||||
return ".tar.bz2"
|
||||
return path.suffix.lower()
|
||||
|
|
@ -0,0 +1,338 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
network_scanner.py
|
||||
"""
|
||||
|
||||
|
||||
from datetime import datetime, UTC
|
||||
from pathlib import Path
|
||||
|
||||
import structlog
|
||||
|
||||
from dlp_scanner.config import ScanConfig
|
||||
from dlp_scanner.detectors.base import DetectorMatch
|
||||
from dlp_scanner.detectors.registry import DetectorRegistry
|
||||
from dlp_scanner.models import (
|
||||
Finding,
|
||||
Location,
|
||||
ScanResult,
|
||||
)
|
||||
from dlp_scanner.network.exfiltration import (
|
||||
DnsExfilDetector,
|
||||
ExfilIndicator,
|
||||
detect_base64_payload,
|
||||
)
|
||||
from dlp_scanner.network.flow_tracker import (
|
||||
FlowTracker,
|
||||
make_flow_key,
|
||||
)
|
||||
from dlp_scanner.network.pcap import read_pcap
|
||||
from dlp_scanner.network.protocols import (
|
||||
DNS_PORT,
|
||||
identify_protocol,
|
||||
parse_dns,
|
||||
parse_http,
|
||||
)
|
||||
from dlp_scanner.scoring import match_to_finding
|
||||
|
||||
|
||||
log = structlog.get_logger()
|
||||
|
||||
EXFIL_RULE_MAP: dict[str, tuple[str, str]] = {
|
||||
"dns_long_label": (
|
||||
"NET_DNS_EXFIL_LONG_LABEL",
|
||||
"DNS Exfiltration: Long Label",
|
||||
),
|
||||
"dns_high_entropy": (
|
||||
"NET_DNS_EXFIL_HIGH_ENTROPY",
|
||||
"DNS Exfiltration: High Entropy Subdomain",
|
||||
),
|
||||
"dns_long_qname": (
|
||||
"NET_DNS_EXFIL_LONG_QNAME",
|
||||
"DNS Exfiltration: Long QNAME",
|
||||
),
|
||||
"dns_txt_volume": (
|
||||
"NET_DNS_EXFIL_TXT_VOLUME",
|
||||
"DNS Exfiltration: High TXT Volume",
|
||||
),
|
||||
"base64_payload": (
|
||||
"NET_ENCODED_BASE64",
|
||||
"Encoded Payload: Base64",
|
||||
),
|
||||
"hex_payload": (
|
||||
"NET_ENCODED_HEX",
|
||||
"Encoded Payload: Hex",
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
class NetworkScanner:
|
||||
"""
|
||||
Scans network capture files for sensitive data in transit
|
||||
"""
|
||||
def __init__(
|
||||
self,
|
||||
config: ScanConfig,
|
||||
registry: DetectorRegistry,
|
||||
) -> None:
|
||||
self._net_config = config.network
|
||||
self._detection_config = config.detection
|
||||
self._redaction_style = config.output.redaction_style
|
||||
self._registry = registry
|
||||
|
||||
def scan(self, target: str) -> ScanResult:
|
||||
"""
|
||||
Scan a PCAP file for sensitive data in payloads
|
||||
"""
|
||||
result = ScanResult()
|
||||
target_path = Path(target)
|
||||
|
||||
if not target_path.exists():
|
||||
result.errors.append(f"PCAP file not found: {target}")
|
||||
result.scan_completed_at = datetime.now(UTC)
|
||||
return result
|
||||
|
||||
try:
|
||||
self._scan_pcap(target_path, result)
|
||||
except Exception as exc:
|
||||
log.warning(
|
||||
"pcap_scan_failed",
|
||||
path = str(target_path),
|
||||
error = str(exc),
|
||||
)
|
||||
result.errors.append(f"PCAP scan failed: {exc}")
|
||||
|
||||
result.scan_completed_at = datetime.now(UTC)
|
||||
return result
|
||||
|
||||
def _scan_pcap(
|
||||
self,
|
||||
path: Path,
|
||||
result: ScanResult,
|
||||
) -> None:
|
||||
"""
|
||||
Read packets, reassemble flows, and run detection
|
||||
"""
|
||||
tracker = FlowTracker()
|
||||
dns_detector = DnsExfilDetector(
|
||||
entropy_threshold = (
|
||||
self._net_config.dns_label_entropy_threshold
|
||||
),
|
||||
)
|
||||
packet_count = 0
|
||||
|
||||
for packet in read_pcap(
|
||||
path,
|
||||
max_packets = self._net_config.max_packets,
|
||||
):
|
||||
packet_count += 1
|
||||
tracker.add_packet(packet)
|
||||
|
||||
if (
|
||||
packet.protocol == "udp"
|
||||
and (
|
||||
packet.src_port == DNS_PORT
|
||||
or packet.dst_port == DNS_PORT
|
||||
)
|
||||
):
|
||||
self._process_dns_packet(
|
||||
packet.payload,
|
||||
packet.src_ip,
|
||||
packet.dst_ip,
|
||||
path,
|
||||
packet_count,
|
||||
dns_detector,
|
||||
result,
|
||||
)
|
||||
|
||||
if packet.payload:
|
||||
exfil_indicators = detect_base64_payload(
|
||||
packet.payload,
|
||||
src_ip = packet.src_ip,
|
||||
dst_ip = packet.dst_ip,
|
||||
)
|
||||
for indicator in exfil_indicators:
|
||||
finding = _indicator_to_finding(
|
||||
indicator,
|
||||
str(path),
|
||||
packet_count,
|
||||
)
|
||||
result.findings.append(finding)
|
||||
|
||||
txt_indicators = dns_detector.check_txt_volume()
|
||||
for indicator in txt_indicators:
|
||||
finding = _indicator_to_finding(
|
||||
indicator,
|
||||
str(path),
|
||||
packet_count,
|
||||
)
|
||||
result.findings.append(finding)
|
||||
|
||||
self._scan_reassembled_flows(tracker, path, result)
|
||||
|
||||
result.targets_scanned = packet_count
|
||||
|
||||
def _process_dns_packet(
|
||||
self,
|
||||
payload: bytes,
|
||||
src_ip: str,
|
||||
dst_ip: str,
|
||||
path: Path,
|
||||
packet_num: int,
|
||||
dns_detector: DnsExfilDetector,
|
||||
result: ScanResult,
|
||||
) -> None:
|
||||
"""
|
||||
Parse DNS and check for exfiltration patterns
|
||||
"""
|
||||
dns_record = parse_dns(payload)
|
||||
if dns_record is None:
|
||||
return
|
||||
|
||||
for query in dns_record.queries:
|
||||
indicator = dns_detector.analyze_query(
|
||||
query,
|
||||
src_ip,
|
||||
dst_ip,
|
||||
)
|
||||
if indicator is not None:
|
||||
finding = _indicator_to_finding(
|
||||
indicator,
|
||||
str(path),
|
||||
packet_num,
|
||||
)
|
||||
result.findings.append(finding)
|
||||
|
||||
def _scan_reassembled_flows(
|
||||
self,
|
||||
tracker: FlowTracker,
|
||||
path: Path,
|
||||
result: ScanResult,
|
||||
) -> None:
|
||||
"""
|
||||
Reassemble TCP streams and scan for sensitive data
|
||||
"""
|
||||
min_confidence = self._detection_config.min_confidence
|
||||
|
||||
for flow in tracker.get_flows():
|
||||
key = (
|
||||
flow.src_ip,
|
||||
flow.dst_ip,
|
||||
flow.src_port,
|
||||
flow.dst_port,
|
||||
)
|
||||
stream = tracker.reassemble_stream(key)
|
||||
if not stream:
|
||||
continue
|
||||
|
||||
protocol = identify_protocol(stream)
|
||||
text = self._extract_scannable_text(
|
||||
stream,
|
||||
protocol,
|
||||
)
|
||||
|
||||
if not text or not text.strip():
|
||||
continue
|
||||
|
||||
matches = self._registry.detect(text)
|
||||
|
||||
location = Location(
|
||||
source_type = "network",
|
||||
uri = str(path),
|
||||
)
|
||||
|
||||
for match in matches:
|
||||
if match.score < min_confidence:
|
||||
continue
|
||||
|
||||
finding = match_to_finding(
|
||||
match,
|
||||
text,
|
||||
location,
|
||||
self._redaction_style,
|
||||
)
|
||||
result.findings.append(finding)
|
||||
|
||||
def _extract_scannable_text(
|
||||
self,
|
||||
stream: bytes,
|
||||
protocol: str,
|
||||
) -> str:
|
||||
"""
|
||||
Extract text content from a reassembled stream
|
||||
"""
|
||||
if protocol == "http":
|
||||
return self._extract_http_text(stream)
|
||||
|
||||
if protocol in ("tls", "ssh"):
|
||||
return ""
|
||||
|
||||
try:
|
||||
return stream.decode("utf-8", errors = "replace")
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
def _extract_http_text(
|
||||
self,
|
||||
stream: bytes,
|
||||
) -> str:
|
||||
"""
|
||||
Extract scannable text from HTTP messages
|
||||
"""
|
||||
http_msg = parse_http(stream)
|
||||
if http_msg is None:
|
||||
try:
|
||||
return stream.decode(
|
||||
"utf-8",
|
||||
errors = "replace",
|
||||
)
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
parts: list[str] = []
|
||||
|
||||
if http_msg.is_request and http_msg.uri:
|
||||
parts.append(http_msg.uri)
|
||||
|
||||
for header_name in ("cookie", "authorization", "set-cookie"):
|
||||
val = http_msg.headers.get(header_name, "")
|
||||
if val:
|
||||
parts.append(val)
|
||||
|
||||
if http_msg.body:
|
||||
parts.append(http_msg.body)
|
||||
|
||||
return "\n".join(parts)
|
||||
|
||||
|
||||
def _indicator_to_finding(
|
||||
indicator: ExfilIndicator,
|
||||
uri: str,
|
||||
packet_num: int,
|
||||
) -> Finding:
|
||||
"""
|
||||
Convert an exfiltration indicator to a Finding
|
||||
"""
|
||||
rule_id, rule_name = EXFIL_RULE_MAP.get(
|
||||
indicator.indicator_type,
|
||||
("NET_EXFIL_UNKNOWN", "Network Exfiltration Indicator"),
|
||||
)
|
||||
|
||||
severity = "high" if indicator.confidence >= 0.70 else "medium"
|
||||
|
||||
location = Location(
|
||||
source_type = "network",
|
||||
uri = uri,
|
||||
byte_offset = packet_num,
|
||||
)
|
||||
|
||||
return Finding(
|
||||
rule_id = rule_id,
|
||||
rule_name = rule_name,
|
||||
severity = severity,
|
||||
confidence = indicator.confidence,
|
||||
location = location,
|
||||
redacted_snippet = indicator.evidence[:120],
|
||||
compliance_frameworks = [],
|
||||
remediation = indicator.description,
|
||||
)
|
||||
|
|
@ -0,0 +1,52 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
scoring.py
|
||||
"""
|
||||
|
||||
|
||||
from dlp_scanner.compliance import (
|
||||
get_frameworks_for_rule,
|
||||
get_remediation_for_rule,
|
||||
score_to_severity,
|
||||
)
|
||||
from dlp_scanner.constants import RedactionStyle
|
||||
from dlp_scanner.detectors.base import DetectorMatch
|
||||
from dlp_scanner.models import Finding, Location
|
||||
from dlp_scanner.redaction import redact
|
||||
|
||||
|
||||
def match_to_finding(
|
||||
match: DetectorMatch,
|
||||
text: str,
|
||||
location: Location,
|
||||
redaction_style: RedactionStyle,
|
||||
) -> Finding:
|
||||
"""
|
||||
Convert a detector match into a fully classified finding
|
||||
"""
|
||||
severity = score_to_severity(match.score)
|
||||
frameworks = get_frameworks_for_rule(match.rule_id)
|
||||
if match.compliance_frameworks:
|
||||
combined = (
|
||||
set(frameworks) | set(match.compliance_frameworks)
|
||||
)
|
||||
frameworks = sorted(combined)
|
||||
remediation = get_remediation_for_rule(match.rule_id)
|
||||
|
||||
snippet = redact(
|
||||
text,
|
||||
match.start,
|
||||
match.end,
|
||||
style = redaction_style,
|
||||
)
|
||||
|
||||
return Finding(
|
||||
rule_id = match.rule_id,
|
||||
rule_name = match.rule_name,
|
||||
severity = severity,
|
||||
confidence = match.score,
|
||||
location = location,
|
||||
redacted_snippet = snippet,
|
||||
compliance_frameworks = frameworks,
|
||||
remediation = remediation,
|
||||
)
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
__init__.py
|
||||
"""
|
||||
|
|
@ -0,0 +1,87 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
conftest.py
|
||||
"""
|
||||
|
||||
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from collections.abc import Generator
|
||||
|
||||
import pytest
|
||||
|
||||
from dlp_scanner.config import ScanConfig
|
||||
from dlp_scanner.models import Finding, Location
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def default_config() -> ScanConfig:
|
||||
"""
|
||||
Provide a default ScanConfig instance
|
||||
"""
|
||||
return ScanConfig()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_location() -> Location:
|
||||
"""
|
||||
Provide a sample file location
|
||||
"""
|
||||
return Location(
|
||||
source_type = "file",
|
||||
uri = "test/employees.csv",
|
||||
line = 42,
|
||||
column = 15,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_finding(sample_location: Location) -> Finding:
|
||||
"""
|
||||
Provide a sample finding
|
||||
"""
|
||||
return Finding(
|
||||
rule_id = "PII_SSN",
|
||||
rule_name = "US Social Security Number",
|
||||
severity = "critical",
|
||||
confidence = 0.95,
|
||||
location = sample_location,
|
||||
redacted_snippet = "...SSN: ***-**-6789...",
|
||||
compliance_frameworks = ["HIPAA",
|
||||
"CCPA"],
|
||||
remediation = "Encrypt or remove SSN data",
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def temp_dir() -> Generator[Path, None, None]:
|
||||
"""
|
||||
Provide a temporary directory for test files
|
||||
"""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
yield Path(tmpdir)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def temp_dir_with_pii(
|
||||
temp_dir: Path,
|
||||
) -> Path:
|
||||
"""
|
||||
Provide a temp directory containing files with known PII
|
||||
"""
|
||||
csv_path = temp_dir / "employees.csv"
|
||||
csv_path.write_text(
|
||||
"name,ssn,email\n"
|
||||
"John Doe,123-45-6789,john@example.com\n"
|
||||
"Jane Smith,987-65-4321,jane@example.com\n"
|
||||
)
|
||||
|
||||
txt_path = temp_dir / "clean.txt"
|
||||
txt_path.write_text("No sensitive data here at all.")
|
||||
|
||||
json_path = temp_dir / "config.json"
|
||||
json_path.write_text(
|
||||
'{"api_key": "sk_live_abc123def456ghi789jkl012mno345"}\n'
|
||||
)
|
||||
|
||||
return temp_dir
|
||||
|
|
@ -0,0 +1,312 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
test_cli.py
|
||||
"""
|
||||
|
||||
|
||||
import json
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from collections.abc import Generator
|
||||
|
||||
import pytest
|
||||
from typer.testing import CliRunner
|
||||
|
||||
from dlp_scanner.cli import app
|
||||
|
||||
|
||||
runner = CliRunner()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def pii_dir() -> Generator[Path, None, None]:
|
||||
"""
|
||||
Provide a temp directory with valid detectable SSNs
|
||||
"""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
root = Path(tmpdir)
|
||||
csv_path = root / "employees.csv"
|
||||
csv_path.write_text(
|
||||
"name,ssn\n"
|
||||
"Alice,456-78-9012\n"
|
||||
"Bob,234-56-7890\n"
|
||||
)
|
||||
yield root
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def json_result_file() -> Generator[Path, None, None]:
|
||||
"""
|
||||
Provide a JSON scan results file for report tests
|
||||
"""
|
||||
data = {
|
||||
"scan_metadata": {
|
||||
"scan_id": "test-123",
|
||||
"tool_version": "0.1.0",
|
||||
"scan_started_at": ("2026-01-01T00:00:00+00:00"),
|
||||
"scan_completed_at": ("2026-01-01T00:01:00+00:00"),
|
||||
"targets_scanned": 1,
|
||||
"total_findings": 1,
|
||||
"errors": [],
|
||||
},
|
||||
"findings": [
|
||||
{
|
||||
"finding_id": "f-001",
|
||||
"rule_id": "PII_SSN",
|
||||
"rule_name": ("US Social Security Number"),
|
||||
"severity": "critical",
|
||||
"confidence": 0.95,
|
||||
"location": {
|
||||
"source_type": "file",
|
||||
"uri": "data.csv",
|
||||
"line": 5,
|
||||
"column": None,
|
||||
"table_name": None,
|
||||
"column_name": None,
|
||||
},
|
||||
"redacted_snippet": "***-**-6789",
|
||||
"compliance_frameworks": [
|
||||
"HIPAA",
|
||||
"CCPA",
|
||||
],
|
||||
"remediation": "Encrypt data",
|
||||
"detected_at": ("2026-01-01T00:00:30+00:00"),
|
||||
}
|
||||
],
|
||||
"summary": {
|
||||
"by_severity": {
|
||||
"critical": 1
|
||||
},
|
||||
"by_rule": {
|
||||
"PII_SSN": 1
|
||||
},
|
||||
"by_framework": {
|
||||
"HIPAA": 1,
|
||||
"CCPA": 1,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
with tempfile.NamedTemporaryFile(
|
||||
suffix = ".json",
|
||||
delete = False,
|
||||
mode = "w",
|
||||
) as f:
|
||||
json.dump(data, f)
|
||||
path = Path(f.name)
|
||||
|
||||
yield path
|
||||
path.unlink(missing_ok = True)
|
||||
|
||||
|
||||
class TestCliHelp:
|
||||
def test_help_shows_commands(self) -> None:
|
||||
result = runner.invoke(app, ["--help"])
|
||||
assert result.exit_code == 0
|
||||
assert "file" in result.output
|
||||
assert "db" in result.output
|
||||
assert "network" in result.output
|
||||
assert "report" in result.output
|
||||
|
||||
def test_version(self) -> None:
|
||||
result = runner.invoke(app, ["--version"])
|
||||
assert result.exit_code == 0
|
||||
assert "0.1.0" in result.output
|
||||
|
||||
def test_file_help(self) -> None:
|
||||
result = runner.invoke(app, ["file", "--help"])
|
||||
assert result.exit_code == 0
|
||||
assert "TARGET" in result.output
|
||||
|
||||
def test_db_help(self) -> None:
|
||||
result = runner.invoke(app, ["db", "--help"])
|
||||
assert result.exit_code == 0
|
||||
assert "TARGET" in result.output
|
||||
|
||||
def test_network_help(self) -> None:
|
||||
result = runner.invoke(app, ["network", "--help"])
|
||||
assert result.exit_code == 0
|
||||
assert "TARGET" in result.output
|
||||
|
||||
def test_report_help(self) -> None:
|
||||
result = runner.invoke(app, ["report", "--help"])
|
||||
assert result.exit_code == 0
|
||||
assert "convert" in result.output
|
||||
assert "summary" in result.output
|
||||
|
||||
|
||||
class TestFileScan:
|
||||
def test_scan_console_output(self, pii_dir: Path) -> None:
|
||||
result = runner.invoke(app, ["file", str(pii_dir)])
|
||||
assert result.exit_code == 0
|
||||
|
||||
def test_scan_json_output(self, pii_dir: Path) -> None:
|
||||
result = runner.invoke(
|
||||
app,
|
||||
["file",
|
||||
str(pii_dir),
|
||||
"-f",
|
||||
"json"],
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
data = json.loads(result.output)
|
||||
assert "findings" in data
|
||||
|
||||
def test_scan_to_file(self, pii_dir: Path) -> None:
|
||||
with tempfile.NamedTemporaryFile(suffix = ".json",
|
||||
delete = False) as f:
|
||||
out_path = f.name
|
||||
|
||||
result = runner.invoke(
|
||||
app,
|
||||
[
|
||||
"file",
|
||||
str(pii_dir),
|
||||
"-f",
|
||||
"json",
|
||||
"-o",
|
||||
out_path,
|
||||
],
|
||||
)
|
||||
assert "Report written" in result.output
|
||||
content = Path(out_path).read_text()
|
||||
assert "findings" in content
|
||||
Path(out_path).unlink(missing_ok = True)
|
||||
|
||||
def test_scan_nonexistent_target(self) -> None:
|
||||
result = runner.invoke(app, ["file", "/no/such/path"])
|
||||
assert result.exit_code == 1
|
||||
|
||||
def test_invalid_format(self, pii_dir: Path) -> None:
|
||||
result = runner.invoke(
|
||||
app,
|
||||
[
|
||||
"file",
|
||||
str(pii_dir),
|
||||
"-f",
|
||||
"invalid",
|
||||
],
|
||||
)
|
||||
assert result.exit_code == 1
|
||||
|
||||
def test_with_config_flag(self, pii_dir: Path) -> None:
|
||||
result = runner.invoke(
|
||||
app,
|
||||
[
|
||||
"--config",
|
||||
"nonexistent.yml",
|
||||
"file",
|
||||
str(pii_dir),
|
||||
],
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
|
||||
def test_with_verbose_flag(self, pii_dir: Path) -> None:
|
||||
result = runner.invoke(
|
||||
app,
|
||||
["--verbose",
|
||||
"file",
|
||||
str(pii_dir)],
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
|
||||
|
||||
class TestReportCommands:
|
||||
def test_convert_to_sarif(self, json_result_file: Path) -> None:
|
||||
result = runner.invoke(
|
||||
app,
|
||||
[
|
||||
"report",
|
||||
"convert",
|
||||
str(json_result_file),
|
||||
"-f",
|
||||
"sarif",
|
||||
],
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
data = json.loads(result.output)
|
||||
assert data["version"] == "2.1.0"
|
||||
|
||||
def test_convert_to_csv(self, json_result_file: Path) -> None:
|
||||
result = runner.invoke(
|
||||
app,
|
||||
[
|
||||
"report",
|
||||
"convert",
|
||||
str(json_result_file),
|
||||
"-f",
|
||||
"csv",
|
||||
],
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
assert "PII_SSN" in result.output
|
||||
|
||||
def test_convert_to_file(self, json_result_file: Path) -> None:
|
||||
with tempfile.NamedTemporaryFile(suffix = ".sarif",
|
||||
delete = False) as f:
|
||||
out_path = f.name
|
||||
|
||||
result = runner.invoke(
|
||||
app,
|
||||
[
|
||||
"report",
|
||||
"convert",
|
||||
str(json_result_file),
|
||||
"-f",
|
||||
"sarif",
|
||||
"-o",
|
||||
out_path,
|
||||
],
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
assert "Converted" in result.output
|
||||
content = Path(out_path).read_text()
|
||||
data = json.loads(content)
|
||||
assert data["version"] == "2.1.0"
|
||||
Path(out_path).unlink(missing_ok = True)
|
||||
|
||||
def test_convert_missing_file(self) -> None:
|
||||
result = runner.invoke(
|
||||
app,
|
||||
[
|
||||
"report",
|
||||
"convert",
|
||||
"/no/such/file.json",
|
||||
],
|
||||
)
|
||||
assert result.exit_code == 1
|
||||
|
||||
def test_convert_invalid_format(self, json_result_file: Path) -> None:
|
||||
result = runner.invoke(
|
||||
app,
|
||||
[
|
||||
"report",
|
||||
"convert",
|
||||
str(json_result_file),
|
||||
"-f",
|
||||
"invalid",
|
||||
],
|
||||
)
|
||||
assert result.exit_code == 1
|
||||
|
||||
def test_summary(self, json_result_file: Path) -> None:
|
||||
result = runner.invoke(
|
||||
app,
|
||||
[
|
||||
"report",
|
||||
"summary",
|
||||
str(json_result_file),
|
||||
],
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
|
||||
def test_summary_missing_file(self) -> None:
|
||||
result = runner.invoke(
|
||||
app,
|
||||
[
|
||||
"report",
|
||||
"summary",
|
||||
"/no/such/file.json",
|
||||
],
|
||||
)
|
||||
assert result.exit_code == 1
|
||||
|
|
@ -0,0 +1,71 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
test_compliance.py
|
||||
"""
|
||||
|
||||
|
||||
from dlp_scanner.compliance import (
|
||||
get_frameworks_for_rule,
|
||||
get_remediation_for_rule,
|
||||
score_to_severity,
|
||||
DEFAULT_REMEDIATION,
|
||||
)
|
||||
|
||||
|
||||
class TestScoreToSeverity:
|
||||
def test_critical_threshold(self) -> None:
|
||||
assert score_to_severity(0.85) == "critical"
|
||||
assert score_to_severity(0.99) == "critical"
|
||||
assert score_to_severity(1.0) == "critical"
|
||||
|
||||
def test_high_threshold(self) -> None:
|
||||
assert score_to_severity(0.65) == "high"
|
||||
assert score_to_severity(0.84) == "high"
|
||||
|
||||
def test_medium_threshold(self) -> None:
|
||||
assert score_to_severity(0.40) == "medium"
|
||||
assert score_to_severity(0.64) == "medium"
|
||||
|
||||
def test_low_threshold(self) -> None:
|
||||
assert score_to_severity(0.20) == "low"
|
||||
assert score_to_severity(0.39) == "low"
|
||||
|
||||
def test_below_minimum(self) -> None:
|
||||
assert score_to_severity(0.19) == "low"
|
||||
assert score_to_severity(0.0) == "low"
|
||||
|
||||
|
||||
class TestFrameworkMapping:
|
||||
def test_ssn_maps_to_hipaa_and_ccpa(self) -> None:
|
||||
frameworks = get_frameworks_for_rule("PII_SSN")
|
||||
assert "HIPAA" in frameworks
|
||||
assert "CCPA" in frameworks
|
||||
assert "GLBA" in frameworks
|
||||
|
||||
def test_credit_card_maps_to_pci(self) -> None:
|
||||
for rule_id in (
|
||||
"FIN_CREDIT_CARD_VISA",
|
||||
"FIN_CREDIT_CARD_MC",
|
||||
"FIN_CREDIT_CARD_AMEX",
|
||||
"FIN_CREDIT_CARD_DISC",
|
||||
):
|
||||
frameworks = get_frameworks_for_rule(rule_id)
|
||||
assert "PCI_DSS" in frameworks
|
||||
|
||||
def test_unknown_rule_returns_empty(self) -> None:
|
||||
assert get_frameworks_for_rule("UNKNOWN") == []
|
||||
|
||||
def test_credential_rules_have_no_frameworks(
|
||||
self,
|
||||
) -> None:
|
||||
frameworks = get_frameworks_for_rule("CRED_AWS_ACCESS_KEY")
|
||||
assert frameworks == []
|
||||
|
||||
|
||||
class TestRemediation:
|
||||
def test_known_rule_has_remediation(self) -> None:
|
||||
text = get_remediation_for_rule("PII_SSN")
|
||||
assert "encrypt" in text.lower() or "tokeniz" in text.lower()
|
||||
|
||||
def test_unknown_rule_returns_default(self) -> None:
|
||||
assert get_remediation_for_rule("UNKNOWN") == DEFAULT_REMEDIATION
|
||||
|
|
@ -0,0 +1,81 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
test_config.py
|
||||
"""
|
||||
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from dlp_scanner.config import ScanConfig, load_config
|
||||
from dlp_scanner.constants import (
|
||||
DEFAULT_DB_SAMPLE_PERCENTAGE,
|
||||
DEFAULT_ENTROPY_THRESHOLD,
|
||||
DEFAULT_MAX_FILE_SIZE_MB,
|
||||
DEFAULT_MIN_CONFIDENCE,
|
||||
)
|
||||
|
||||
|
||||
class TestScanConfig:
|
||||
def test_defaults(self) -> None:
|
||||
config = ScanConfig()
|
||||
assert config.file.max_file_size_mb == DEFAULT_MAX_FILE_SIZE_MB
|
||||
assert config.file.recursive is True
|
||||
assert config.database.sample_percentage == DEFAULT_DB_SAMPLE_PERCENTAGE
|
||||
assert config.network.entropy_threshold == DEFAULT_ENTROPY_THRESHOLD
|
||||
assert config.detection.min_confidence == DEFAULT_MIN_CONFIDENCE
|
||||
assert config.output.format == "console"
|
||||
assert config.output.redaction_style == "partial"
|
||||
|
||||
def test_exclude_patterns_populated(self) -> None:
|
||||
config = ScanConfig()
|
||||
assert "*.pyc" in config.file.exclude_patterns
|
||||
assert ".git" in config.file.exclude_patterns
|
||||
|
||||
def test_include_extensions_populated(self) -> None:
|
||||
config = ScanConfig()
|
||||
assert ".pdf" in config.file.include_extensions
|
||||
assert ".csv" in config.file.include_extensions
|
||||
|
||||
def test_default_frameworks(self) -> None:
|
||||
config = ScanConfig()
|
||||
assert "HIPAA" in config.compliance.frameworks
|
||||
assert "PCI_DSS" in config.compliance.frameworks
|
||||
|
||||
|
||||
class TestLoadConfig:
|
||||
def test_load_missing_file_returns_defaults(self) -> None:
|
||||
config = load_config(Path("/nonexistent/config.yml"))
|
||||
assert config == ScanConfig()
|
||||
|
||||
def test_load_none_returns_defaults(self) -> None:
|
||||
config = load_config(None)
|
||||
assert isinstance(config, ScanConfig)
|
||||
|
||||
def test_load_yaml_config(self, tmp_path: Path) -> None:
|
||||
config_path = tmp_path / ".dlp-scanner.yml"
|
||||
config_path.write_text(
|
||||
"scan:\n"
|
||||
" file:\n"
|
||||
" max_file_size_mb: 50\n"
|
||||
" recursive: false\n"
|
||||
"detection:\n"
|
||||
" min_confidence: 0.5\n"
|
||||
"output:\n"
|
||||
" format: json\n"
|
||||
)
|
||||
config = load_config(config_path)
|
||||
assert config.file.max_file_size_mb == 50
|
||||
assert config.file.recursive is False
|
||||
assert config.detection.min_confidence == 0.5
|
||||
assert config.output.format == "json"
|
||||
|
||||
def test_load_partial_config_fills_defaults(
|
||||
self,
|
||||
tmp_path: Path
|
||||
) -> None:
|
||||
config_path = tmp_path / ".dlp-scanner.yml"
|
||||
config_path.write_text("output:\n format: sarif\n")
|
||||
config = load_config(config_path)
|
||||
assert config.output.format == "sarif"
|
||||
assert config.file.max_file_size_mb == DEFAULT_MAX_FILE_SIZE_MB
|
||||
assert config.detection.min_confidence == DEFAULT_MIN_CONFIDENCE
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
__init__.py
|
||||
"""
|
||||
|
|
@ -0,0 +1,120 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
test_context.py
|
||||
"""
|
||||
|
||||
|
||||
from dlp_scanner.detectors.base import DetectorMatch
|
||||
from dlp_scanner.detectors.context import (
|
||||
apply_context_boost,
|
||||
_apply_cooccurrence_boost,
|
||||
)
|
||||
|
||||
|
||||
def _make_match(
|
||||
rule_id: str = "PII_SSN",
|
||||
start: int = 20,
|
||||
end: int = 31,
|
||||
score: float = 0.45,
|
||||
keywords: list[str] | None = None,
|
||||
) -> DetectorMatch:
|
||||
resolved_keywords = (
|
||||
keywords if keywords is not None else ["ssn",
|
||||
"social security"]
|
||||
)
|
||||
return DetectorMatch(
|
||||
rule_id = rule_id,
|
||||
rule_name = "Test Rule",
|
||||
start = start,
|
||||
end = end,
|
||||
matched_text = "234-56-7890",
|
||||
score = score,
|
||||
context_keywords = resolved_keywords,
|
||||
compliance_frameworks = [],
|
||||
)
|
||||
|
||||
|
||||
class TestContextBoost:
|
||||
def test_boost_with_keyword_present(self) -> None:
|
||||
text = "Employee SSN: 234-56-7890 on file"
|
||||
match = _make_match(start = 14, end = 25)
|
||||
boosted = apply_context_boost(text, [match])
|
||||
assert boosted[0].score > match.score
|
||||
|
||||
def test_no_boost_without_keyword(self) -> None:
|
||||
text = "Some random number 234-56-7890 here"
|
||||
match = _make_match(
|
||||
start = 19,
|
||||
end = 30,
|
||||
keywords = ["nonexistent_keyword"],
|
||||
)
|
||||
boosted = apply_context_boost(text, [match])
|
||||
assert boosted[0].score == match.score
|
||||
|
||||
def test_no_boost_with_empty_keywords(self) -> None:
|
||||
text = "SSN: 234-56-7890"
|
||||
match = _make_match(start = 5, end = 16, keywords = [])
|
||||
boosted = apply_context_boost(text, [match])
|
||||
assert boosted[0].score == match.score
|
||||
|
||||
def test_empty_matches_returns_empty(self) -> None:
|
||||
result = apply_context_boost("any text", [])
|
||||
assert result == []
|
||||
|
||||
|
||||
class TestCooccurrenceBoost:
|
||||
def test_nearby_different_rules_boosted(self) -> None:
|
||||
matches = [
|
||||
_make_match(rule_id = "PII_SSN",
|
||||
start = 10,
|
||||
end = 21),
|
||||
_make_match(
|
||||
rule_id = "PII_EMAIL",
|
||||
start = 30,
|
||||
end = 50,
|
||||
keywords = ["email"],
|
||||
),
|
||||
]
|
||||
boosted = _apply_cooccurrence_boost(matches)
|
||||
assert all(
|
||||
b.score > m.score
|
||||
for b, m in zip(boosted, matches, strict = False)
|
||||
)
|
||||
|
||||
def test_same_rule_not_boosted(self) -> None:
|
||||
matches = [
|
||||
_make_match(rule_id = "PII_SSN",
|
||||
start = 10,
|
||||
end = 21),
|
||||
_make_match(rule_id = "PII_SSN",
|
||||
start = 50,
|
||||
end = 61),
|
||||
]
|
||||
boosted = _apply_cooccurrence_boost(matches)
|
||||
assert all(
|
||||
b.score == m.score
|
||||
for b, m in zip(boosted, matches, strict = False)
|
||||
)
|
||||
|
||||
def test_distant_matches_not_boosted(self) -> None:
|
||||
matches = [
|
||||
_make_match(rule_id = "PII_SSN",
|
||||
start = 10,
|
||||
end = 21),
|
||||
_make_match(
|
||||
rule_id = "PII_EMAIL",
|
||||
start = 1000,
|
||||
end = 1020,
|
||||
keywords = ["email"],
|
||||
),
|
||||
]
|
||||
boosted = _apply_cooccurrence_boost(matches)
|
||||
assert all(
|
||||
b.score == m.score
|
||||
for b, m in zip(boosted, matches, strict = False)
|
||||
)
|
||||
|
||||
def test_single_match_not_boosted(self) -> None:
|
||||
matches = [_make_match()]
|
||||
boosted = _apply_cooccurrence_boost(matches)
|
||||
assert boosted[0].score == matches[0].score
|
||||
|
|
@ -0,0 +1,83 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
test_entropy.py
|
||||
"""
|
||||
|
||||
|
||||
import os
|
||||
|
||||
from dlp_scanner.detectors.entropy import (
|
||||
shannon_entropy,
|
||||
shannon_entropy_str,
|
||||
detect_high_entropy_regions,
|
||||
EntropyDetector,
|
||||
)
|
||||
|
||||
|
||||
class TestShannonEntropy:
|
||||
def test_all_same_bytes_is_zero(self) -> None:
|
||||
data = b"\x00" * 100
|
||||
assert shannon_entropy(data) == 0.0
|
||||
|
||||
def test_empty_data_is_zero(self) -> None:
|
||||
assert shannon_entropy(b"") == 0.0
|
||||
|
||||
def test_english_text_in_expected_range(self) -> None:
|
||||
text = b"the quick brown fox jumps over the lazy dog"
|
||||
h = shannon_entropy(text)
|
||||
assert 3.5 <= h <= 5.0
|
||||
|
||||
def test_random_bytes_near_maximum(self) -> None:
|
||||
data = os.urandom(10000)
|
||||
h = shannon_entropy(data)
|
||||
assert h > 7.5
|
||||
|
||||
def test_two_byte_values_is_one_bit(self) -> None:
|
||||
data = b"\x00\x01" * 50
|
||||
h = shannon_entropy(data)
|
||||
assert abs(h - 1.0) < 0.01
|
||||
|
||||
def test_string_entropy_matches_bytes(self) -> None:
|
||||
text = "hello world"
|
||||
h_str = shannon_entropy_str(text)
|
||||
h_bytes = shannon_entropy(text.encode("utf-8"))
|
||||
assert abs(h_str - h_bytes) < 0.001
|
||||
|
||||
|
||||
class TestHighEntropyRegions:
|
||||
def test_random_data_detected(self) -> None:
|
||||
data = os.urandom(1024)
|
||||
regions = detect_high_entropy_regions(data, threshold = 7.0)
|
||||
assert len(regions) > 0
|
||||
|
||||
def test_plaintext_not_detected(self) -> None:
|
||||
data = b"the quick brown fox " * 100
|
||||
regions = detect_high_entropy_regions(data, threshold = 7.0)
|
||||
assert len(regions) == 0
|
||||
|
||||
def test_short_data_below_window(self) -> None:
|
||||
data = os.urandom(100)
|
||||
regions = detect_high_entropy_regions(
|
||||
data,
|
||||
threshold = 7.0,
|
||||
window_size = 256
|
||||
)
|
||||
assert len(regions) <= 1
|
||||
|
||||
|
||||
class TestEntropyDetector:
|
||||
def test_detect_high_entropy_text(self) -> None:
|
||||
import base64
|
||||
|
||||
detector = EntropyDetector(threshold = 5.5)
|
||||
raw = os.urandom(2048)
|
||||
high_entropy_text = base64.b85encode(raw).decode("ascii")
|
||||
matches = detector.detect(high_entropy_text)
|
||||
assert len(matches) > 0
|
||||
assert all(m.rule_id == "NET_HIGH_ENTROPY" for m in matches)
|
||||
|
||||
def test_no_detection_in_normal_text(self) -> None:
|
||||
detector = EntropyDetector(threshold = 7.0)
|
||||
text = "This is a normal text document with nothing suspicious."
|
||||
matches = detector.detect(text)
|
||||
assert len(matches) == 0
|
||||
|
|
@ -0,0 +1,58 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
test_pattern.py
|
||||
"""
|
||||
|
||||
|
||||
from dlp_scanner.detectors.pattern import PatternDetector
|
||||
from dlp_scanner.detectors.rules.pii import PII_RULES
|
||||
|
||||
|
||||
class TestPatternDetector:
|
||||
def test_detects_ssn_in_text(self) -> None:
|
||||
detector = PatternDetector(
|
||||
rules = PII_RULES,
|
||||
allowlist_values = frozenset()
|
||||
)
|
||||
text = "Employee SSN is 234-56-7890 on file."
|
||||
matches = detector.detect(text)
|
||||
ssn_matches = [m for m in matches if m.rule_id == "PII_SSN"]
|
||||
assert len(ssn_matches) == 1
|
||||
assert ssn_matches[0].matched_text == "234-56-7890"
|
||||
|
||||
def test_skips_allowlisted_values(self) -> None:
|
||||
detector = PatternDetector(rules = PII_RULES)
|
||||
text = "Test SSN: 123-45-6789"
|
||||
matches = detector.detect(text)
|
||||
ssn_matches = [m for m in matches if m.rule_id == "PII_SSN"]
|
||||
assert len(ssn_matches) == 0
|
||||
|
||||
def test_detects_email(self) -> None:
|
||||
detector = PatternDetector(
|
||||
rules = PII_RULES,
|
||||
allowlist_values = frozenset()
|
||||
)
|
||||
text = "Contact: alice@company.com for details."
|
||||
matches = detector.detect(text)
|
||||
email_matches = [m for m in matches if m.rule_id == "PII_EMAIL"]
|
||||
assert len(email_matches) == 1
|
||||
|
||||
def test_no_matches_in_clean_text(self) -> None:
|
||||
detector = PatternDetector(rules = PII_RULES)
|
||||
text = "This is a perfectly clean document."
|
||||
matches = detector.detect(text)
|
||||
assert len(matches) == 0
|
||||
|
||||
def test_multiple_matches_in_one_text(self) -> None:
|
||||
detector = PatternDetector(
|
||||
rules = PII_RULES,
|
||||
allowlist_values = frozenset()
|
||||
)
|
||||
text = (
|
||||
"Name: John, SSN: 234-56-7890, "
|
||||
"Email: john@test.org, Phone: (555) 234-5678"
|
||||
)
|
||||
matches = detector.detect(text)
|
||||
rule_ids = {m.rule_id for m in matches}
|
||||
assert "PII_SSN" in rule_ids
|
||||
assert "PII_EMAIL" in rule_ids
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
__init__.py
|
||||
"""
|
||||
|
|
@ -0,0 +1,123 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
test_credentials.py
|
||||
"""
|
||||
|
||||
|
||||
import pytest
|
||||
|
||||
from dlp_scanner.detectors.rules.credentials import (
|
||||
AWS_ACCESS_KEY_PATTERN,
|
||||
GITHUB_CLASSIC_PAT_PATTERN,
|
||||
GITHUB_FINE_GRAINED_PATTERN,
|
||||
JWT_PATTERN,
|
||||
STRIPE_KEY_PATTERN,
|
||||
SLACK_TOKEN_PATTERN,
|
||||
PRIVATE_KEY_PATTERN,
|
||||
GENERIC_API_KEY_PATTERN,
|
||||
)
|
||||
|
||||
|
||||
class TestAWSAccessKey:
|
||||
def test_long_term_key_matches(self) -> None:
|
||||
assert (
|
||||
AWS_ACCESS_KEY_PATTERN.search("AKIAIOSFODNN7EXAMPLE")
|
||||
is not None
|
||||
)
|
||||
|
||||
def test_session_key_matches(self) -> None:
|
||||
assert (
|
||||
AWS_ACCESS_KEY_PATTERN.search("ASIAQWERTYUIOP123456")
|
||||
is not None
|
||||
)
|
||||
|
||||
def test_invalid_prefix_rejected(self) -> None:
|
||||
assert (
|
||||
AWS_ACCESS_KEY_PATTERN.search("ABCDIOSFODNN7EXAMPLE") is None
|
||||
)
|
||||
|
||||
|
||||
class TestGitHubTokens:
|
||||
def test_classic_pat_matches(self) -> None:
|
||||
token = "ghp_" + "a" * 36
|
||||
assert (GITHUB_CLASSIC_PAT_PATTERN.search(token) is not None)
|
||||
|
||||
def test_fine_grained_pat_matches(self) -> None:
|
||||
token = "github_pat_" + "a" * 22 + "_" + "b" * 59
|
||||
assert (GITHUB_FINE_GRAINED_PATTERN.search(token) is not None)
|
||||
|
||||
def test_invalid_prefix_rejected(self) -> None:
|
||||
assert (
|
||||
GITHUB_CLASSIC_PAT_PATTERN.search("xyz_" + "a" * 36) is None
|
||||
)
|
||||
|
||||
|
||||
class TestJWT:
|
||||
def test_jwt_matches(self) -> None:
|
||||
token = (
|
||||
"eyJhbGciOiJIUzI1NiJ9"
|
||||
".eyJzdWIiOiIxMjM0NTY3ODkwIn0"
|
||||
".abc123def456"
|
||||
)
|
||||
assert JWT_PATTERN.search(token) is not None
|
||||
|
||||
def test_non_jwt_rejected(self) -> None:
|
||||
assert (JWT_PATTERN.search("not.a.jwt.token") is None)
|
||||
|
||||
|
||||
class TestStripeKey:
|
||||
@pytest.mark.parametrize(
|
||||
"key",
|
||||
[
|
||||
"sk_test_" + "a" * 24,
|
||||
"sk_live_" + "b" * 24,
|
||||
"pk_test_" + "c" * 24,
|
||||
"pk_live_" + "d" * 30,
|
||||
],
|
||||
)
|
||||
def test_stripe_keys_match(self, key: str) -> None:
|
||||
assert STRIPE_KEY_PATTERN.search(key) is not None
|
||||
|
||||
def test_invalid_stripe_key(self) -> None:
|
||||
assert (STRIPE_KEY_PATTERN.search("sk_invalid_abc") is None)
|
||||
|
||||
|
||||
class TestSlackToken:
|
||||
@pytest.mark.parametrize(
|
||||
"token",
|
||||
[
|
||||
"xoxb-" + "a" * 20,
|
||||
"xoxp-" + "b" * 30,
|
||||
"xoxa-" + "c" * 15,
|
||||
],
|
||||
)
|
||||
def test_slack_tokens_match(self, token: str) -> None:
|
||||
assert (SLACK_TOKEN_PATTERN.search(token) is not None)
|
||||
|
||||
|
||||
class TestPrivateKey:
|
||||
@pytest.mark.parametrize(
|
||||
"header",
|
||||
[
|
||||
"-----BEGIN RSA PRIVATE KEY-----",
|
||||
"-----BEGIN EC PRIVATE KEY-----",
|
||||
"-----BEGIN PRIVATE KEY-----",
|
||||
"-----BEGIN OPENSSH PRIVATE KEY-----",
|
||||
],
|
||||
)
|
||||
def test_private_key_headers_match(self, header: str) -> None:
|
||||
assert (PRIVATE_KEY_PATTERN.search(header) is not None)
|
||||
|
||||
|
||||
class TestGenericAPIKey:
|
||||
@pytest.mark.parametrize(
|
||||
"text",
|
||||
[
|
||||
'api_key = "abcdef1234567890abcdef"',
|
||||
"API_KEY: abcdef1234567890abcdef",
|
||||
"secret_key='very_secret_key_value_12345'",
|
||||
'access_key = "abc123def456ghi789jkl012"',
|
||||
],
|
||||
)
|
||||
def test_generic_api_keys_match(self, text: str) -> None:
|
||||
assert (GENERIC_API_KEY_PATTERN.search(text) is not None)
|
||||
|
|
@ -0,0 +1,125 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
test_financial.py
|
||||
"""
|
||||
|
||||
|
||||
import pytest
|
||||
|
||||
from dlp_scanner.detectors.rules.financial import (
|
||||
VISA_PATTERN,
|
||||
MASTERCARD_PATTERN,
|
||||
AMEX_PATTERN,
|
||||
IBAN_PATTERN,
|
||||
luhn_check,
|
||||
iban_check,
|
||||
nhs_check,
|
||||
)
|
||||
|
||||
|
||||
class TestLuhnAlgorithm:
|
||||
@pytest.mark.parametrize(
|
||||
"number",
|
||||
[
|
||||
"4532015112830366",
|
||||
"4916338506082832",
|
||||
"5425233430109903",
|
||||
"2223000048410010",
|
||||
"374245455400126",
|
||||
"6011000990139424",
|
||||
],
|
||||
)
|
||||
def test_valid_cards_pass_luhn(self, number: str) -> None:
|
||||
assert luhn_check(number) is True
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"number",
|
||||
[
|
||||
"4532015112830367",
|
||||
"1234567890123456",
|
||||
"1111111111111112",
|
||||
"9999999999999991",
|
||||
],
|
||||
)
|
||||
def test_invalid_cards_fail_luhn(self, number: str) -> None:
|
||||
assert luhn_check(number) is False
|
||||
|
||||
def test_too_short_fails(self) -> None:
|
||||
assert luhn_check("123456") is False
|
||||
|
||||
def test_with_spaces(self) -> None:
|
||||
assert luhn_check("4532 0151 1283 0366") is True
|
||||
|
||||
def test_with_dashes(self) -> None:
|
||||
assert luhn_check("4532-0151-1283-0366") is True
|
||||
|
||||
|
||||
class TestIBANCheck:
|
||||
@pytest.mark.parametrize(
|
||||
"iban",
|
||||
[
|
||||
"GB29NWBK60161331926819",
|
||||
"DE89370400440532013000",
|
||||
"FR7630006000011234567890189",
|
||||
"NL91ABNA0417164300",
|
||||
],
|
||||
)
|
||||
def test_valid_ibans(self, iban: str) -> None:
|
||||
assert iban_check(iban) is True
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"iban",
|
||||
[
|
||||
"GB29NWBK60161331926818",
|
||||
"XX00INVALID",
|
||||
"DE00000000000000000000",
|
||||
"SHORT",
|
||||
],
|
||||
)
|
||||
def test_invalid_ibans(self, iban: str) -> None:
|
||||
assert iban_check(iban) is False
|
||||
|
||||
def test_iban_with_spaces(self) -> None:
|
||||
assert iban_check("GB29 NWBK 6016 1331 9268 19") is True
|
||||
|
||||
|
||||
class TestNHSCheck:
|
||||
def test_valid_nhs_number(self) -> None:
|
||||
assert nhs_check("9434765919") is True
|
||||
|
||||
def test_invalid_nhs_number(self) -> None:
|
||||
assert nhs_check("1234567890") is False
|
||||
|
||||
def test_nhs_too_short(self) -> None:
|
||||
assert nhs_check("12345") is False
|
||||
|
||||
def test_nhs_non_numeric(self) -> None:
|
||||
assert nhs_check("abcdefghij") is False
|
||||
|
||||
|
||||
class TestCreditCardPatterns:
|
||||
def test_visa_pattern_matches(self) -> None:
|
||||
assert VISA_PATTERN.search("4532015112830366") is not None
|
||||
|
||||
def test_mastercard_classic_matches(self) -> None:
|
||||
assert MASTERCARD_PATTERN.search("5425233430109903") is not None
|
||||
|
||||
def test_mastercard_2series_matches(self) -> None:
|
||||
assert MASTERCARD_PATTERN.search("2223000048410010") is not None
|
||||
|
||||
def test_amex_matches(self) -> None:
|
||||
assert AMEX_PATTERN.search("374245455400126") is not None
|
||||
|
||||
def test_visa_with_spaces(self) -> None:
|
||||
assert VISA_PATTERN.search("4532 0151 1283 0366") is not None
|
||||
|
||||
def test_visa_with_dashes(self) -> None:
|
||||
assert VISA_PATTERN.search("4532-0151-1283-0366") is not None
|
||||
|
||||
|
||||
class TestIBANPattern:
|
||||
def test_iban_pattern_matches_gb(self) -> None:
|
||||
assert IBAN_PATTERN.search("GB29NWBK60161331926819") is not None
|
||||
|
||||
def test_iban_pattern_matches_de(self) -> None:
|
||||
assert IBAN_PATTERN.search("DE89370400440532013000") is not None
|
||||
|
|
@ -0,0 +1,112 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
test_health.py
|
||||
"""
|
||||
|
||||
|
||||
import pytest
|
||||
|
||||
from dlp_scanner.detectors.rules.health import (
|
||||
MEDICAL_RECORD_PATTERN,
|
||||
DEA_NUMBER_PATTERN,
|
||||
NPI_PATTERN,
|
||||
_validate_dea_number,
|
||||
_validate_npi,
|
||||
)
|
||||
|
||||
|
||||
class TestMedicalRecordPattern:
|
||||
@pytest.mark.parametrize(
|
||||
"text",
|
||||
[
|
||||
"MRN: 123456",
|
||||
"MRN:12345678",
|
||||
"MR# 9876543210",
|
||||
"MED-1234567890",
|
||||
"mrn 00123456",
|
||||
],
|
||||
)
|
||||
def test_valid_mrns_match(self, text: str) -> None:
|
||||
assert (
|
||||
MEDICAL_RECORD_PATTERN.search(text) is not None
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"text",
|
||||
[
|
||||
"MRN: 12345",
|
||||
"MORNING coffee",
|
||||
"random text without MRN",
|
||||
],
|
||||
)
|
||||
def test_invalid_mrns_rejected(self, text: str) -> None:
|
||||
assert (
|
||||
MEDICAL_RECORD_PATTERN.search(text) is None
|
||||
)
|
||||
|
||||
|
||||
class TestDEANumberPattern:
|
||||
def test_valid_dea_format_matches(self) -> None:
|
||||
assert (
|
||||
DEA_NUMBER_PATTERN.search("AB1234563") is not None
|
||||
)
|
||||
|
||||
def test_lowercase_rejected(self) -> None:
|
||||
assert (
|
||||
DEA_NUMBER_PATTERN.search("ab1234563") is None
|
||||
)
|
||||
|
||||
def test_too_short_rejected(self) -> None:
|
||||
assert (
|
||||
DEA_NUMBER_PATTERN.search("AB12345") is None
|
||||
)
|
||||
|
||||
|
||||
class TestDEAValidation:
|
||||
def test_valid_dea_number(self) -> None:
|
||||
assert _validate_dea_number("AB1234563") is True
|
||||
|
||||
def test_invalid_check_digit(self) -> None:
|
||||
assert _validate_dea_number("AB1234560") is False
|
||||
|
||||
def test_too_short(self) -> None:
|
||||
assert _validate_dea_number("AB12345") is False
|
||||
|
||||
def test_non_numeric_digits(self) -> None:
|
||||
assert _validate_dea_number("ABabcdefg") is False
|
||||
|
||||
def test_valid_with_9_prefix(self) -> None:
|
||||
assert _validate_dea_number("A91234563") is True
|
||||
|
||||
|
||||
class TestNPIPattern:
|
||||
def test_ten_digit_matches(self) -> None:
|
||||
assert NPI_PATTERN.search("1234567890") is not None
|
||||
|
||||
def test_nine_digit_rejected(self) -> None:
|
||||
assert NPI_PATTERN.search("123456789") is None
|
||||
|
||||
def test_eleven_digit_no_exact_match(self) -> None:
|
||||
match = NPI_PATTERN.search("12345678901")
|
||||
if match is not None:
|
||||
assert len(match.group()) == 10
|
||||
|
||||
|
||||
class TestNPIValidation:
|
||||
def test_valid_npi(self) -> None:
|
||||
assert _validate_npi("1234567893") is True
|
||||
|
||||
def test_invalid_check_digit(self) -> None:
|
||||
assert _validate_npi("1234567890") is False
|
||||
|
||||
def test_non_numeric(self) -> None:
|
||||
assert _validate_npi("abcdefghij") is False
|
||||
|
||||
def test_too_short(self) -> None:
|
||||
assert _validate_npi("12345") is False
|
||||
|
||||
def test_valid_npi_second(self) -> None:
|
||||
assert _validate_npi("1679576722") is True
|
||||
|
||||
def test_all_zeros_invalid(self) -> None:
|
||||
assert _validate_npi("0000000000") is False
|
||||
|
|
@ -0,0 +1,138 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
test_pii.py
|
||||
"""
|
||||
|
||||
|
||||
import pytest
|
||||
|
||||
from dlp_scanner.detectors.rules.pii import (
|
||||
SSN_PATTERN,
|
||||
EMAIL_PATTERN,
|
||||
PHONE_US_PATTERN,
|
||||
IPV4_PATTERN,
|
||||
_validate_ssn,
|
||||
)
|
||||
|
||||
|
||||
class TestSSNPattern:
|
||||
@pytest.mark.parametrize(
|
||||
"text",
|
||||
[
|
||||
"234-56-7890",
|
||||
"567-89-0123",
|
||||
"001-01-0001",
|
||||
"899-99-9999",
|
||||
],
|
||||
)
|
||||
def test_valid_ssns_match(self, text: str) -> None:
|
||||
assert SSN_PATTERN.search(text) is not None
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"text",
|
||||
[
|
||||
"000-45-6789",
|
||||
"666-45-6789",
|
||||
"900-45-6789",
|
||||
"999-45-6789",
|
||||
"123-00-6789",
|
||||
"123-45-0000",
|
||||
],
|
||||
)
|
||||
def test_invalid_ssns_rejected(self, text: str) -> None:
|
||||
match = SSN_PATTERN.search(text)
|
||||
if match is not None:
|
||||
assert not _validate_ssn(match.group())
|
||||
|
||||
|
||||
class TestSSNValidation:
|
||||
def test_valid_ssn(self) -> None:
|
||||
assert _validate_ssn("234-56-7890") is True
|
||||
|
||||
def test_invalid_area_000(self) -> None:
|
||||
assert _validate_ssn("000-45-6789") is False
|
||||
|
||||
def test_invalid_area_666(self) -> None:
|
||||
assert _validate_ssn("666-45-6789") is False
|
||||
|
||||
def test_invalid_area_900_plus(self) -> None:
|
||||
assert _validate_ssn("950-45-6789") is False
|
||||
|
||||
def test_invalid_group_00(self) -> None:
|
||||
assert _validate_ssn("123-00-6789") is False
|
||||
|
||||
def test_invalid_serial_0000(self) -> None:
|
||||
assert _validate_ssn("123-45-0000") is False
|
||||
|
||||
def test_bare_format(self) -> None:
|
||||
assert _validate_ssn("234567890") is True
|
||||
|
||||
def test_non_numeric(self) -> None:
|
||||
assert _validate_ssn("abc-de-fghi") is False
|
||||
|
||||
|
||||
class TestEmailPattern:
|
||||
@pytest.mark.parametrize(
|
||||
"text",
|
||||
[
|
||||
"user@example.com",
|
||||
"first.last@company.org",
|
||||
"user+tag@domain.co.uk",
|
||||
"test_email@test.museum",
|
||||
],
|
||||
)
|
||||
def test_valid_emails_match(self, text: str) -> None:
|
||||
assert EMAIL_PATTERN.search(text) is not None
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"text",
|
||||
[
|
||||
"not-an-email",
|
||||
"@nodomain",
|
||||
"user@",
|
||||
"user@.com",
|
||||
],
|
||||
)
|
||||
def test_invalid_emails_rejected(self, text: str) -> None:
|
||||
assert EMAIL_PATTERN.search(text) is None
|
||||
|
||||
|
||||
class TestPhoneUSPattern:
|
||||
@pytest.mark.parametrize(
|
||||
"text",
|
||||
[
|
||||
"(555) 234-5678",
|
||||
"555-234-5678",
|
||||
"555.234.5678",
|
||||
"+1 555-234-5678",
|
||||
"1-555-234-5678",
|
||||
],
|
||||
)
|
||||
def test_valid_phones_match(self, text: str) -> None:
|
||||
assert PHONE_US_PATTERN.search(text) is not None
|
||||
|
||||
|
||||
class TestIPv4Pattern:
|
||||
@pytest.mark.parametrize(
|
||||
"text",
|
||||
[
|
||||
"192.168.1.1",
|
||||
"10.0.0.1",
|
||||
"255.255.255.255",
|
||||
"0.0.0.0",
|
||||
"172.16.0.1",
|
||||
],
|
||||
)
|
||||
def test_valid_ips_match(self, text: str) -> None:
|
||||
assert IPV4_PATTERN.search(text) is not None
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"text",
|
||||
[
|
||||
"256.1.1.1",
|
||||
"1.1.1.256",
|
||||
"999.999.999.999",
|
||||
],
|
||||
)
|
||||
def test_invalid_ips_rejected(self, text: str) -> None:
|
||||
assert IPV4_PATTERN.search(text) is None
|
||||
|
|
@ -0,0 +1,220 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
test_engine.py
|
||||
"""
|
||||
|
||||
|
||||
import json
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from collections.abc import Generator
|
||||
|
||||
import pytest
|
||||
|
||||
from dlp_scanner.config import ScanConfig
|
||||
from dlp_scanner.engine import ScanEngine
|
||||
from dlp_scanner.models import (
|
||||
Finding,
|
||||
Location,
|
||||
ScanResult,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def engine() -> ScanEngine:
|
||||
"""
|
||||
Provide a ScanEngine with default config
|
||||
"""
|
||||
return ScanEngine(ScanConfig())
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def pii_dir() -> Generator[Path, None, None]:
|
||||
"""
|
||||
Provide a temp directory with valid detectable SSNs
|
||||
"""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
root = Path(tmpdir)
|
||||
csv_path = root / "employees.csv"
|
||||
csv_path.write_text(
|
||||
"name,ssn\n"
|
||||
"Alice,456-78-9012\n"
|
||||
"Bob,234-56-7890\n"
|
||||
)
|
||||
yield root
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def clean_dir() -> Generator[Path, None, None]:
|
||||
"""
|
||||
Provide a temp directory with no sensitive data
|
||||
"""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
root = Path(tmpdir)
|
||||
txt_path = root / "readme.txt"
|
||||
txt_path.write_text("This file contains no sensitive data.")
|
||||
yield root
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def result_with_findings() -> ScanResult:
|
||||
"""
|
||||
Provide a ScanResult with test findings
|
||||
"""
|
||||
result = ScanResult(targets_scanned = 1)
|
||||
result.findings = [
|
||||
Finding(
|
||||
rule_id = "PII_SSN",
|
||||
rule_name = "US Social Security Number",
|
||||
severity = "critical",
|
||||
confidence = 0.95,
|
||||
location = Location(
|
||||
source_type = "file",
|
||||
uri = "data.csv",
|
||||
line = 5,
|
||||
),
|
||||
redacted_snippet = "***-**-6789",
|
||||
compliance_frameworks = ["HIPAA",
|
||||
"CCPA"],
|
||||
remediation = "Encrypt data",
|
||||
),
|
||||
]
|
||||
return result
|
||||
|
||||
|
||||
class TestScanEngine:
|
||||
def test_scan_files_finds_pii(
|
||||
self,
|
||||
engine: ScanEngine,
|
||||
pii_dir: Path
|
||||
) -> None:
|
||||
result = engine.scan_files(str(pii_dir))
|
||||
assert len(result.findings) > 0
|
||||
|
||||
def test_scan_files_clean_dir(
|
||||
self,
|
||||
engine: ScanEngine,
|
||||
clean_dir: Path
|
||||
) -> None:
|
||||
result = engine.scan_files(str(clean_dir))
|
||||
assert len(result.findings) == 0
|
||||
|
||||
def test_scan_files_nonexistent(self, engine: ScanEngine) -> None:
|
||||
result = engine.scan_files("/no/such/path")
|
||||
assert len(result.errors) > 0
|
||||
|
||||
def test_scan_files_sets_completed_at(
|
||||
self,
|
||||
engine: ScanEngine,
|
||||
pii_dir: Path
|
||||
) -> None:
|
||||
result = engine.scan_files(str(pii_dir))
|
||||
assert result.scan_completed_at is not None
|
||||
|
||||
def test_scan_database_sqlite(self, engine: ScanEngine) -> None:
|
||||
with tempfile.NamedTemporaryFile(suffix = ".db",
|
||||
delete = False) as f:
|
||||
db_path = f.name
|
||||
|
||||
import sqlite3
|
||||
|
||||
conn = sqlite3.connect(db_path)
|
||||
conn.execute("CREATE TABLE users "
|
||||
"(name TEXT, ssn TEXT)")
|
||||
conn.execute(
|
||||
"INSERT INTO users VALUES "
|
||||
"('Alice', '456-78-9012')"
|
||||
)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
uri = f"sqlite:///{db_path}"
|
||||
result = engine.scan_database(uri)
|
||||
assert len(result.findings) > 0
|
||||
|
||||
Path(db_path).unlink(missing_ok = True)
|
||||
|
||||
def test_generate_report_json(
|
||||
self,
|
||||
engine: ScanEngine,
|
||||
result_with_findings: ScanResult,
|
||||
) -> None:
|
||||
output = engine.generate_report(result_with_findings, "json")
|
||||
data = json.loads(output)
|
||||
assert "findings" in data
|
||||
assert len(data["findings"]) == 1
|
||||
|
||||
def test_generate_report_sarif(
|
||||
self,
|
||||
engine: ScanEngine,
|
||||
result_with_findings: ScanResult,
|
||||
) -> None:
|
||||
output = engine.generate_report(result_with_findings, "sarif")
|
||||
data = json.loads(output)
|
||||
assert data["version"] == "2.1.0"
|
||||
|
||||
def test_generate_report_csv(
|
||||
self,
|
||||
engine: ScanEngine,
|
||||
result_with_findings: ScanResult,
|
||||
) -> None:
|
||||
output = engine.generate_report(result_with_findings, "csv")
|
||||
lines = output.strip().split("\n")
|
||||
assert len(lines) == 2
|
||||
|
||||
def test_generate_report_console(
|
||||
self,
|
||||
engine: ScanEngine,
|
||||
result_with_findings: ScanResult,
|
||||
) -> None:
|
||||
output = engine.generate_report(result_with_findings, "console")
|
||||
assert "PII_SSN" in output or "Social" in output
|
||||
|
||||
def test_generate_report_uses_config_default(
|
||||
self,
|
||||
result_with_findings: ScanResult,
|
||||
) -> None:
|
||||
config = ScanConfig()
|
||||
config.output.format = "json"
|
||||
engine = ScanEngine(config)
|
||||
output = engine.generate_report(result_with_findings)
|
||||
data = json.loads(output)
|
||||
assert "findings" in data
|
||||
|
||||
def test_display_console(
|
||||
self,
|
||||
engine: ScanEngine,
|
||||
result_with_findings: ScanResult,
|
||||
) -> None:
|
||||
engine.display_console(result_with_findings)
|
||||
|
||||
def test_write_report(
|
||||
self,
|
||||
engine: ScanEngine,
|
||||
result_with_findings: ScanResult,
|
||||
) -> None:
|
||||
with tempfile.NamedTemporaryFile(
|
||||
suffix = ".json",
|
||||
delete = False,
|
||||
mode = "w",
|
||||
) as f:
|
||||
output_path = f.name
|
||||
|
||||
engine.write_report(
|
||||
result_with_findings,
|
||||
output_path,
|
||||
"json",
|
||||
)
|
||||
content = Path(output_path).read_text()
|
||||
data = json.loads(content)
|
||||
assert len(data["findings"]) == 1
|
||||
|
||||
Path(output_path).unlink(missing_ok = True)
|
||||
|
||||
|
||||
class TestReporterMap:
|
||||
def test_all_formats_have_reporters(self) -> None:
|
||||
from dlp_scanner.engine import REPORTER_MAP
|
||||
|
||||
expected = {"console", "json", "sarif", "csv"}
|
||||
assert set(REPORTER_MAP.keys()) == expected
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
__init__.py
|
||||
"""
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
__init__.py
|
||||
"""
|
||||
|
|
@ -0,0 +1,160 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
test_exfiltration.py
|
||||
"""
|
||||
|
||||
|
||||
from dlp_scanner.network.exfiltration import (
|
||||
DnsExfilDetector,
|
||||
_extract_base_domain,
|
||||
detect_base64_payload,
|
||||
)
|
||||
from dlp_scanner.network.protocols import DnsQuery
|
||||
|
||||
|
||||
class TestExtractBaseDomain:
|
||||
def test_simple_domain(self) -> None:
|
||||
assert (_extract_base_domain("www.example.com") == "example.com")
|
||||
|
||||
def test_deep_subdomain(self) -> None:
|
||||
result = _extract_base_domain("a.b.c.example.com")
|
||||
assert result == "example.com"
|
||||
|
||||
def test_trailing_dot(self) -> None:
|
||||
result = _extract_base_domain("www.example.com.")
|
||||
assert result == "example.com"
|
||||
|
||||
def test_single_label(self) -> None:
|
||||
assert _extract_base_domain("localhost") == ("localhost")
|
||||
|
||||
def test_two_labels(self) -> None:
|
||||
assert (_extract_base_domain("example.com") == "example.com")
|
||||
|
||||
|
||||
class TestDnsExfilDetector:
|
||||
def test_normal_query_no_indicator(self) -> None:
|
||||
detector = DnsExfilDetector()
|
||||
query = DnsQuery(
|
||||
name = "www.google.com",
|
||||
query_type = "A",
|
||||
query_class = "1",
|
||||
)
|
||||
result = detector.analyze_query(query, "10.0.0.1", "8.8.8.8")
|
||||
assert result is None
|
||||
|
||||
def test_long_label_detected(self) -> None:
|
||||
detector = DnsExfilDetector()
|
||||
long_label = "a" * 55
|
||||
query = DnsQuery(
|
||||
name = f"{long_label}.evil.com",
|
||||
query_type = "A",
|
||||
query_class = "1",
|
||||
)
|
||||
result = detector.analyze_query(query, "10.0.0.1", "1.2.3.4")
|
||||
assert result is not None
|
||||
assert (result.indicator_type == "dns_long_label")
|
||||
|
||||
def test_high_entropy_subdomain(self) -> None:
|
||||
detector = DnsExfilDetector(entropy_threshold = 3.5)
|
||||
encoded = "aGVsbG8gd29ybGQgdGhpcw"
|
||||
query = DnsQuery(
|
||||
name = f"{encoded}.evil.com",
|
||||
query_type = "A",
|
||||
query_class = "1",
|
||||
)
|
||||
result = detector.analyze_query(query, "10.0.0.1", "1.2.3.4")
|
||||
assert result is not None
|
||||
assert (result.indicator_type == "dns_high_entropy")
|
||||
|
||||
def test_long_qname_detected(self) -> None:
|
||||
detector = DnsExfilDetector()
|
||||
parts = ["abc"] * 40
|
||||
name = ".".join(parts) + ".evil.com"
|
||||
query = DnsQuery(
|
||||
name = name,
|
||||
query_type = "A",
|
||||
query_class = "1",
|
||||
)
|
||||
result = detector.analyze_query(query, "10.0.0.1", "1.2.3.4")
|
||||
assert result is not None
|
||||
|
||||
def test_txt_volume_detection(self) -> None:
|
||||
detector = DnsExfilDetector()
|
||||
for _ in range(10):
|
||||
detector.analyze_query(
|
||||
DnsQuery(
|
||||
name = "data.evil.com",
|
||||
query_type = "TXT",
|
||||
query_class = "1",
|
||||
),
|
||||
"10.0.0.1",
|
||||
"1.2.3.4",
|
||||
)
|
||||
|
||||
indicators = detector.check_txt_volume()
|
||||
assert len(indicators) > 0
|
||||
assert (indicators[0].indicator_type == "dns_txt_volume")
|
||||
|
||||
def test_get_indicators_accumulates(
|
||||
self,
|
||||
) -> None:
|
||||
detector = DnsExfilDetector()
|
||||
long_label = "x" * 55
|
||||
query = DnsQuery(
|
||||
name = f"{long_label}.evil.com",
|
||||
query_type = "A",
|
||||
query_class = "1",
|
||||
)
|
||||
detector.analyze_query(query, "10.0.0.1", "1.2.3.4")
|
||||
detector.analyze_query(query, "10.0.0.1", "1.2.3.4")
|
||||
|
||||
indicators = detector.get_indicators()
|
||||
assert len(indicators) == 2
|
||||
|
||||
def test_short_subdomain_no_entropy_check(
|
||||
self,
|
||||
) -> None:
|
||||
detector = DnsExfilDetector(entropy_threshold = 3.0)
|
||||
query = DnsQuery(
|
||||
name = "example.com",
|
||||
query_type = "A",
|
||||
query_class = "1",
|
||||
)
|
||||
result = detector.analyze_query(query, "10.0.0.1", "8.8.8.8")
|
||||
assert result is None
|
||||
|
||||
|
||||
class TestDetectBase64Payload:
|
||||
def test_base64_detected(self) -> None:
|
||||
payload = (b"data=" + b"A" * 50 + b"== end")
|
||||
indicators = detect_base64_payload(payload)
|
||||
assert len(indicators) > 0
|
||||
assert (indicators[0].indicator_type == "base64_payload")
|
||||
|
||||
def test_hex_detected(self) -> None:
|
||||
payload = b"0x" + b"aabbccdd" * 10
|
||||
indicators = detect_base64_payload(payload)
|
||||
assert len(indicators) > 0
|
||||
types = {i.indicator_type for i in indicators}
|
||||
assert "hex_payload" in types
|
||||
|
||||
def test_normal_text_no_detection(self) -> None:
|
||||
payload = b"Hello, this is normal text."
|
||||
indicators = detect_base64_payload(payload)
|
||||
assert len(indicators) == 0
|
||||
|
||||
def test_short_base64_not_detected(self) -> None:
|
||||
payload = b"dGVzdA=="
|
||||
indicators = detect_base64_payload(payload)
|
||||
assert len(indicators) == 0
|
||||
|
||||
def test_source_ip_preserved(self) -> None:
|
||||
payload = b"A" * 50
|
||||
indicators = detect_base64_payload(
|
||||
payload,
|
||||
src_ip = "10.0.0.1",
|
||||
dst_ip = "1.2.3.4",
|
||||
)
|
||||
assert len(indicators) > 0
|
||||
assert indicators[0].source_ip == "10.0.0.1"
|
||||
assert indicators[0].dest_ip == "1.2.3.4"
|
||||
|
|
@ -0,0 +1,195 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
test_flow_tracker.py
|
||||
"""
|
||||
|
||||
|
||||
from dlp_scanner.network.flow_tracker import (
|
||||
FlowTracker,
|
||||
make_flow_key,
|
||||
)
|
||||
from dlp_scanner.network.pcap import PacketInfo
|
||||
|
||||
|
||||
def _make_packet(
|
||||
src_ip: str = "192.168.1.1",
|
||||
dst_ip: str = "10.0.0.1",
|
||||
src_port: int = 12345,
|
||||
dst_port: int = 80,
|
||||
protocol: str = "tcp",
|
||||
payload: bytes = b"data",
|
||||
timestamp: float = 1.0,
|
||||
tcp_seq: int = 0,
|
||||
) -> PacketInfo:
|
||||
"""
|
||||
Helper to create a PacketInfo for testing
|
||||
"""
|
||||
return PacketInfo(
|
||||
timestamp = timestamp,
|
||||
src_ip = src_ip,
|
||||
dst_ip = dst_ip,
|
||||
src_port = src_port,
|
||||
dst_port = dst_port,
|
||||
protocol = protocol,
|
||||
payload = payload,
|
||||
raw_length = len(payload) + 54,
|
||||
tcp_seq = tcp_seq,
|
||||
)
|
||||
|
||||
|
||||
class TestMakeFlowKey:
|
||||
def test_bidirectional_key(self) -> None:
|
||||
pkt_fwd = _make_packet(
|
||||
src_ip = "192.168.1.1",
|
||||
dst_ip = "10.0.0.1",
|
||||
src_port = 12345,
|
||||
dst_port = 80,
|
||||
)
|
||||
pkt_rev = _make_packet(
|
||||
src_ip = "10.0.0.1",
|
||||
dst_ip = "192.168.1.1",
|
||||
src_port = 80,
|
||||
dst_port = 12345,
|
||||
)
|
||||
assert make_flow_key(pkt_fwd) == make_flow_key(pkt_rev)
|
||||
|
||||
def test_different_ports_different_key(
|
||||
self,
|
||||
) -> None:
|
||||
pkt1 = _make_packet(src_port = 1000)
|
||||
pkt2 = _make_packet(src_port = 2000)
|
||||
assert make_flow_key(pkt1) != make_flow_key(pkt2)
|
||||
|
||||
|
||||
class TestFlowTracker:
|
||||
def test_add_single_packet(self) -> None:
|
||||
tracker = FlowTracker()
|
||||
pkt = _make_packet()
|
||||
tracker.add_packet(pkt)
|
||||
|
||||
assert tracker.flow_count == 1
|
||||
flows = tracker.get_flows()
|
||||
assert flows[0].packet_count == 1
|
||||
assert flows[0].total_bytes == 4
|
||||
|
||||
def test_add_multiple_packets_same_flow(
|
||||
self,
|
||||
) -> None:
|
||||
tracker = FlowTracker()
|
||||
pkt1 = _make_packet(timestamp = 1.0)
|
||||
pkt2 = _make_packet(timestamp = 2.0)
|
||||
tracker.add_packet(pkt1)
|
||||
tracker.add_packet(pkt2)
|
||||
|
||||
assert tracker.flow_count == 1
|
||||
flow = tracker.get_flows()[0]
|
||||
assert flow.packet_count == 2
|
||||
assert flow.total_bytes == 8
|
||||
assert flow.start_time == 1.0
|
||||
assert flow.end_time == 2.0
|
||||
|
||||
def test_different_flows_tracked(self) -> None:
|
||||
tracker = FlowTracker()
|
||||
pkt1 = _make_packet(dst_port = 80)
|
||||
pkt2 = _make_packet(dst_port = 443)
|
||||
tracker.add_packet(pkt1)
|
||||
tracker.add_packet(pkt2)
|
||||
|
||||
assert tracker.flow_count == 2
|
||||
|
||||
def test_bidirectional_packets_same_flow(
|
||||
self,
|
||||
) -> None:
|
||||
tracker = FlowTracker()
|
||||
pkt_out = _make_packet(
|
||||
src_ip = "192.168.1.1",
|
||||
dst_ip = "10.0.0.1",
|
||||
)
|
||||
pkt_in = _make_packet(
|
||||
src_ip = "10.0.0.1",
|
||||
dst_ip = "192.168.1.1",
|
||||
src_port = 80,
|
||||
dst_port = 12345,
|
||||
)
|
||||
tracker.add_packet(pkt_out)
|
||||
tracker.add_packet(pkt_in)
|
||||
|
||||
assert tracker.flow_count == 1
|
||||
flow = tracker.get_flows()[0]
|
||||
assert flow.packet_count == 2
|
||||
|
||||
def test_reassemble_stream_ordered(
|
||||
self,
|
||||
) -> None:
|
||||
tracker = FlowTracker()
|
||||
pkt1 = _make_packet(
|
||||
payload = b"first",
|
||||
tcp_seq = 100,
|
||||
timestamp = 1.0,
|
||||
)
|
||||
pkt2 = _make_packet(
|
||||
payload = b"second",
|
||||
tcp_seq = 200,
|
||||
timestamp = 2.0,
|
||||
)
|
||||
pkt3 = _make_packet(
|
||||
payload = b"third",
|
||||
tcp_seq = 150,
|
||||
timestamp = 1.5,
|
||||
)
|
||||
tracker.add_packet(pkt1)
|
||||
tracker.add_packet(pkt2)
|
||||
tracker.add_packet(pkt3)
|
||||
|
||||
key = make_flow_key(pkt1)
|
||||
stream = tracker.reassemble_stream(key)
|
||||
assert stream == b"firstthirdsecond"
|
||||
|
||||
def test_reassemble_deduplicates_retransmits(
|
||||
self,
|
||||
) -> None:
|
||||
tracker = FlowTracker()
|
||||
pkt1 = _make_packet(
|
||||
payload = b"data",
|
||||
tcp_seq = 100,
|
||||
)
|
||||
pkt2 = _make_packet(
|
||||
payload = b"data",
|
||||
tcp_seq = 100,
|
||||
)
|
||||
tracker.add_packet(pkt1)
|
||||
tracker.add_packet(pkt2)
|
||||
|
||||
key = make_flow_key(pkt1)
|
||||
stream = tracker.reassemble_stream(key)
|
||||
assert stream == b"data"
|
||||
|
||||
def test_reassemble_unknown_key(self) -> None:
|
||||
tracker = FlowTracker()
|
||||
result = tracker.reassemble_stream(("1.1.1.1", "2.2.2.2", 1, 2))
|
||||
assert result == b""
|
||||
|
||||
def test_get_flow_by_key(self) -> None:
|
||||
tracker = FlowTracker()
|
||||
pkt = _make_packet()
|
||||
tracker.add_packet(pkt)
|
||||
|
||||
key = make_flow_key(pkt)
|
||||
flow = tracker.get_flow(key)
|
||||
assert flow is not None
|
||||
assert flow.packet_count == 1
|
||||
|
||||
def test_get_flow_missing_key(self) -> None:
|
||||
tracker = FlowTracker()
|
||||
flow = tracker.get_flow(("1.1.1.1", "2.2.2.2", 0, 0))
|
||||
assert flow is None
|
||||
|
||||
def test_empty_payload_not_stored(self) -> None:
|
||||
tracker = FlowTracker()
|
||||
pkt = _make_packet(payload = b"")
|
||||
tracker.add_packet(pkt)
|
||||
|
||||
key = make_flow_key(pkt)
|
||||
flow = tracker.get_flow(key)
|
||||
assert flow is not None
|
||||
assert len(flow.segments) == 0
|
||||
|
|
@ -0,0 +1,60 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
test_pcap.py
|
||||
"""
|
||||
|
||||
|
||||
from dlp_scanner.network.pcap import PacketInfo
|
||||
|
||||
|
||||
class TestPacketInfo:
|
||||
def test_tcp_packet_construction(self) -> None:
|
||||
pkt = PacketInfo(
|
||||
timestamp = 1000.0,
|
||||
src_ip = "192.168.1.1",
|
||||
dst_ip = "10.0.0.1",
|
||||
src_port = 12345,
|
||||
dst_port = 80,
|
||||
protocol = "tcp",
|
||||
payload = b"hello",
|
||||
raw_length = 100,
|
||||
tcp_flags = 0x02,
|
||||
tcp_seq = 1000,
|
||||
)
|
||||
assert pkt.src_ip == "192.168.1.1"
|
||||
assert pkt.dst_ip == "10.0.0.1"
|
||||
assert pkt.protocol == "tcp"
|
||||
assert pkt.payload == b"hello"
|
||||
assert pkt.tcp_seq == 1000
|
||||
|
||||
def test_udp_packet_defaults(self) -> None:
|
||||
pkt = PacketInfo(
|
||||
timestamp = 1000.0,
|
||||
src_ip = "10.0.0.1",
|
||||
dst_ip = "8.8.8.8",
|
||||
src_port = 54321,
|
||||
dst_port = 53,
|
||||
protocol = "udp",
|
||||
payload = b"\x00",
|
||||
raw_length = 50,
|
||||
)
|
||||
assert pkt.tcp_flags == 0
|
||||
assert pkt.tcp_seq == 0
|
||||
assert pkt.protocol == "udp"
|
||||
|
||||
def test_packet_is_frozen(self) -> None:
|
||||
pkt = PacketInfo(
|
||||
timestamp = 1.0,
|
||||
src_ip = "1.1.1.1",
|
||||
dst_ip = "2.2.2.2",
|
||||
src_port = 1,
|
||||
dst_port = 2,
|
||||
protocol = "tcp",
|
||||
payload = b"",
|
||||
raw_length = 0,
|
||||
)
|
||||
try:
|
||||
pkt.src_ip = "changed"
|
||||
raise AssertionError()
|
||||
except AttributeError:
|
||||
pass
|
||||
|
|
@ -0,0 +1,158 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
test_protocols.py
|
||||
"""
|
||||
|
||||
|
||||
from dlp_scanner.network.protocols import (
|
||||
_is_http_request,
|
||||
_parse_txt_rdata,
|
||||
identify_protocol,
|
||||
parse_dns,
|
||||
parse_http,
|
||||
)
|
||||
|
||||
|
||||
class TestIdentifyProtocol:
|
||||
def test_http_get_request(self) -> None:
|
||||
payload = b"GET / HTTP/1.1\r\nHost: x\r\n\r\n"
|
||||
assert identify_protocol(payload) == "http"
|
||||
|
||||
def test_http_post_request(self) -> None:
|
||||
payload = b"POST /api HTTP/1.1\r\n\r\n"
|
||||
assert identify_protocol(payload) == "http"
|
||||
|
||||
def test_http_response(self) -> None:
|
||||
payload = b"HTTP/1.1 200 OK\r\n\r\n"
|
||||
assert identify_protocol(payload) == "http"
|
||||
|
||||
def test_tls_handshake(self) -> None:
|
||||
payload = b"\x16\x03\x01\x00\x05hello"
|
||||
assert identify_protocol(payload) == "tls"
|
||||
|
||||
def test_ssh_banner(self) -> None:
|
||||
payload = b"SSH-2.0-OpenSSH_8.9\r\n"
|
||||
assert identify_protocol(payload) == "ssh"
|
||||
|
||||
def test_smtp_banner(self) -> None:
|
||||
payload = b"220 mail.example.com ESMTP"
|
||||
assert identify_protocol(payload) == "smtp"
|
||||
|
||||
def test_unknown_protocol(self) -> None:
|
||||
payload = b"\x00\x01\x02\x03"
|
||||
assert identify_protocol(payload) == "unknown"
|
||||
|
||||
def test_empty_payload(self) -> None:
|
||||
assert identify_protocol(b"") == "unknown"
|
||||
|
||||
|
||||
class TestIsHttpRequest:
|
||||
def test_get_is_http(self) -> None:
|
||||
assert _is_http_request(b"GET /path HTTP/1.1")
|
||||
|
||||
def test_delete_is_http(self) -> None:
|
||||
assert _is_http_request(b"DELETE /resource HTTP/1.1")
|
||||
|
||||
def test_random_bytes_not_http(self) -> None:
|
||||
assert not _is_http_request(b"\x00\x01\x02")
|
||||
|
||||
def test_short_payload_not_http(self) -> None:
|
||||
assert not _is_http_request(b"HI")
|
||||
|
||||
|
||||
class TestParseHttp:
|
||||
def test_parse_get_request(self) -> None:
|
||||
raw = (
|
||||
b"GET /index.html HTTP/1.1\r\n"
|
||||
b"Host: example.com\r\n"
|
||||
b"\r\n"
|
||||
)
|
||||
result = parse_http(raw)
|
||||
assert result is not None
|
||||
assert result.method == "GET"
|
||||
assert result.uri == "/index.html"
|
||||
assert result.is_request is True
|
||||
assert "host" in result.headers
|
||||
|
||||
def test_parse_post_with_body(self) -> None:
|
||||
body = b"key=value"
|
||||
raw = (
|
||||
b"POST /api HTTP/1.1\r\n"
|
||||
b"Content-Length: 9\r\n"
|
||||
b"\r\n" + body
|
||||
)
|
||||
result = parse_http(raw)
|
||||
assert result is not None
|
||||
assert result.method == "POST"
|
||||
assert result.body == "key=value"
|
||||
|
||||
def test_parse_response(self) -> None:
|
||||
raw = (
|
||||
b"HTTP/1.1 200 OK\r\n"
|
||||
b"Content-Type: text/html\r\n"
|
||||
b"Content-Length: 5\r\n"
|
||||
b"\r\nhello"
|
||||
)
|
||||
result = parse_http(raw)
|
||||
assert result is not None
|
||||
assert result.is_request is False
|
||||
assert result.body == "hello"
|
||||
|
||||
def test_invalid_data_returns_none(self) -> None:
|
||||
assert parse_http(b"\x00\x01") is None
|
||||
|
||||
|
||||
class TestParseDns:
|
||||
def test_parse_dns_query(self) -> None:
|
||||
import dpkt
|
||||
|
||||
dns = dpkt.dns.DNS()
|
||||
dns.id = 1234
|
||||
dns.qr = dpkt.dns.DNS_Q
|
||||
dns.opcode = dpkt.dns.DNS_QUERY
|
||||
q = dpkt.dns.DNS.Q()
|
||||
q.name = "example.com"
|
||||
q.type = dpkt.dns.DNS_A
|
||||
q.cls = dpkt.dns.DNS_IN
|
||||
dns.qd = [q]
|
||||
|
||||
result = parse_dns(bytes(dns))
|
||||
assert result is not None
|
||||
assert len(result.queries) == 1
|
||||
assert result.queries[0].name == "example.com"
|
||||
assert result.queries[0].query_type == "A"
|
||||
assert result.is_response is False
|
||||
assert result.transaction_id == 1234
|
||||
|
||||
def test_parse_txt_query(self) -> None:
|
||||
import dpkt
|
||||
|
||||
dns = dpkt.dns.DNS()
|
||||
dns.id = 5678
|
||||
dns.qr = dpkt.dns.DNS_Q
|
||||
q = dpkt.dns.DNS.Q()
|
||||
q.name = "data.evil.com"
|
||||
q.type = dpkt.dns.DNS_TXT
|
||||
q.cls = dpkt.dns.DNS_IN
|
||||
dns.qd = [q]
|
||||
|
||||
result = parse_dns(bytes(dns))
|
||||
assert result is not None
|
||||
assert result.queries[0].query_type == "TXT"
|
||||
assert (result.queries[0].name == "data.evil.com")
|
||||
|
||||
def test_invalid_data_returns_none(self) -> None:
|
||||
assert parse_dns(b"\x00\x01") is None
|
||||
|
||||
|
||||
class TestParseTxtRdata:
|
||||
def test_single_string(self) -> None:
|
||||
rdata = b"\x05hello"
|
||||
assert _parse_txt_rdata(rdata) == "hello"
|
||||
|
||||
def test_multiple_strings(self) -> None:
|
||||
rdata = b"\x02hi\x05world"
|
||||
assert _parse_txt_rdata(rdata) == "hi world"
|
||||
|
||||
def test_empty_rdata(self) -> None:
|
||||
assert _parse_txt_rdata(b"") == ""
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
__init__.py
|
||||
"""
|
||||
|
|
@ -0,0 +1,104 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
test_csv_report.py
|
||||
"""
|
||||
|
||||
|
||||
import csv
|
||||
import io
|
||||
|
||||
import pytest
|
||||
|
||||
from dlp_scanner.models import (
|
||||
Finding,
|
||||
Location,
|
||||
ScanResult,
|
||||
)
|
||||
from dlp_scanner.reporters.csv_report import (
|
||||
CSV_COLUMNS,
|
||||
CsvReporter,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def result_with_findings() -> ScanResult:
|
||||
"""
|
||||
Provide a ScanResult with test findings
|
||||
"""
|
||||
result = ScanResult(targets_scanned = 1)
|
||||
result.findings = [
|
||||
Finding(
|
||||
rule_id = "PII_SSN",
|
||||
rule_name = "US Social Security Number",
|
||||
severity = "critical",
|
||||
confidence = 0.95,
|
||||
location = Location(
|
||||
source_type = "file",
|
||||
uri = "data.csv",
|
||||
line = 5,
|
||||
),
|
||||
redacted_snippet = "***-**-6789",
|
||||
compliance_frameworks = [
|
||||
"HIPAA",
|
||||
"CCPA",
|
||||
],
|
||||
remediation = "Encrypt data",
|
||||
),
|
||||
]
|
||||
return result
|
||||
|
||||
|
||||
class TestCsvReporter:
|
||||
def test_generates_valid_csv(
|
||||
self,
|
||||
result_with_findings: ScanResult
|
||||
) -> None:
|
||||
reporter = CsvReporter()
|
||||
output = reporter.generate(result_with_findings)
|
||||
reader = csv.reader(io.StringIO(output))
|
||||
rows = list(reader)
|
||||
assert len(rows) == 2
|
||||
|
||||
def test_header_matches_columns(
|
||||
self,
|
||||
result_with_findings: ScanResult
|
||||
) -> None:
|
||||
reporter = CsvReporter()
|
||||
output = reporter.generate(result_with_findings)
|
||||
reader = csv.reader(io.StringIO(output))
|
||||
header = next(reader)
|
||||
assert header == CSV_COLUMNS
|
||||
|
||||
def test_finding_row_data(
|
||||
self,
|
||||
result_with_findings: ScanResult
|
||||
) -> None:
|
||||
reporter = CsvReporter()
|
||||
output = reporter.generate(result_with_findings)
|
||||
reader = csv.reader(io.StringIO(output))
|
||||
next(reader)
|
||||
row = next(reader)
|
||||
assert row[2] == "critical"
|
||||
assert row[4] == "PII_SSN"
|
||||
assert row[7] == "data.csv"
|
||||
assert "HIPAA" in row[12]
|
||||
assert "CCPA" in row[12]
|
||||
|
||||
def test_empty_result(self) -> None:
|
||||
reporter = CsvReporter()
|
||||
result = ScanResult()
|
||||
output = reporter.generate(result)
|
||||
reader = csv.reader(io.StringIO(output))
|
||||
rows = list(reader)
|
||||
assert len(rows) == 1
|
||||
|
||||
def test_frameworks_semicolon_separated(
|
||||
self,
|
||||
result_with_findings: ScanResult
|
||||
) -> None:
|
||||
reporter = CsvReporter()
|
||||
output = reporter.generate(result_with_findings)
|
||||
reader = csv.reader(io.StringIO(output))
|
||||
next(reader)
|
||||
row = next(reader)
|
||||
assert row[12] == "HIPAA;CCPA"
|
||||
|
|
@ -0,0 +1,159 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
test_json_report.py
|
||||
"""
|
||||
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from dlp_scanner.models import (
|
||||
Finding,
|
||||
Location,
|
||||
ScanResult,
|
||||
)
|
||||
from dlp_scanner.reporters.json_report import (
|
||||
JsonReporter,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def result_with_findings() -> ScanResult:
|
||||
"""
|
||||
Provide a ScanResult with test findings
|
||||
"""
|
||||
result = ScanResult(targets_scanned = 3)
|
||||
result.findings = [
|
||||
Finding(
|
||||
rule_id = "PII_SSN",
|
||||
rule_name = "US Social Security Number",
|
||||
severity = "critical",
|
||||
confidence = 0.95,
|
||||
location = Location(
|
||||
source_type = "file",
|
||||
uri = "employees.csv",
|
||||
line = 2,
|
||||
),
|
||||
redacted_snippet = "SSN: ***-**-6789",
|
||||
compliance_frameworks = [
|
||||
"HIPAA",
|
||||
"CCPA",
|
||||
],
|
||||
remediation = "Encrypt SSN data",
|
||||
),
|
||||
Finding(
|
||||
rule_id = "PII_EMAIL",
|
||||
rule_name = "Email Address",
|
||||
severity = "medium",
|
||||
confidence = 0.65,
|
||||
location = Location(
|
||||
source_type = "file",
|
||||
uri = "contacts.json",
|
||||
),
|
||||
redacted_snippet = "j***@example.com",
|
||||
compliance_frameworks = ["GDPR"],
|
||||
remediation = "Hash emails",
|
||||
),
|
||||
]
|
||||
return result
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def empty_result() -> ScanResult:
|
||||
"""
|
||||
Provide a ScanResult with no findings
|
||||
"""
|
||||
return ScanResult(targets_scanned = 5)
|
||||
|
||||
|
||||
class TestJsonReporter:
|
||||
def test_generates_valid_json(
|
||||
self,
|
||||
result_with_findings: ScanResult
|
||||
) -> None:
|
||||
reporter = JsonReporter()
|
||||
output = reporter.generate(result_with_findings)
|
||||
data = json.loads(output)
|
||||
assert isinstance(data, dict)
|
||||
|
||||
def test_has_metadata_section(
|
||||
self,
|
||||
result_with_findings: ScanResult
|
||||
) -> None:
|
||||
reporter = JsonReporter()
|
||||
output = reporter.generate(result_with_findings)
|
||||
data = json.loads(output)
|
||||
meta = data["scan_metadata"]
|
||||
assert meta["scan_id"]
|
||||
assert meta["tool_version"] == "0.1.0"
|
||||
assert meta["targets_scanned"] == 3
|
||||
assert meta["total_findings"] == 2
|
||||
|
||||
def test_has_findings_section(
|
||||
self,
|
||||
result_with_findings: ScanResult
|
||||
) -> None:
|
||||
reporter = JsonReporter()
|
||||
output = reporter.generate(result_with_findings)
|
||||
data = json.loads(output)
|
||||
findings = data["findings"]
|
||||
assert len(findings) == 2
|
||||
assert findings[0]["rule_id"] == "PII_SSN"
|
||||
assert findings[0]["severity"] == "critical"
|
||||
assert findings[0]["confidence"] == 0.95
|
||||
|
||||
def test_finding_has_location(
|
||||
self,
|
||||
result_with_findings: ScanResult
|
||||
) -> None:
|
||||
reporter = JsonReporter()
|
||||
output = reporter.generate(result_with_findings)
|
||||
data = json.loads(output)
|
||||
loc = data["findings"][0]["location"]
|
||||
assert loc["source_type"] == "file"
|
||||
assert loc["uri"] == "employees.csv"
|
||||
assert loc["line"] == 2
|
||||
|
||||
def test_has_summary_section(
|
||||
self,
|
||||
result_with_findings: ScanResult
|
||||
) -> None:
|
||||
reporter = JsonReporter()
|
||||
output = reporter.generate(result_with_findings)
|
||||
data = json.loads(output)
|
||||
summary = data["summary"]
|
||||
assert summary["by_severity"]["critical"] == 1
|
||||
assert summary["by_severity"]["medium"] == 1
|
||||
assert summary["by_rule"]["PII_SSN"] == 1
|
||||
assert summary["by_framework"]["HIPAA"] == 1
|
||||
|
||||
def test_empty_result_has_zero_findings(
|
||||
self,
|
||||
empty_result: ScanResult
|
||||
) -> None:
|
||||
reporter = JsonReporter()
|
||||
output = reporter.generate(empty_result)
|
||||
data = json.loads(output)
|
||||
assert len(data["findings"]) == 0
|
||||
assert (data["scan_metadata"]["total_findings"] == 0)
|
||||
|
||||
def test_finding_has_remediation(
|
||||
self,
|
||||
result_with_findings: ScanResult
|
||||
) -> None:
|
||||
reporter = JsonReporter()
|
||||
output = reporter.generate(result_with_findings)
|
||||
data = json.loads(output)
|
||||
assert (data["findings"][0]["remediation"] == "Encrypt SSN data")
|
||||
|
||||
def test_finding_has_compliance(
|
||||
self,
|
||||
result_with_findings: ScanResult
|
||||
) -> None:
|
||||
reporter = JsonReporter()
|
||||
output = reporter.generate(result_with_findings)
|
||||
data = json.loads(output)
|
||||
frameworks = data["findings"][0]["compliance_frameworks"]
|
||||
assert "HIPAA" in frameworks
|
||||
assert "CCPA" in frameworks
|
||||
|
|
@ -0,0 +1,174 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
test_sarif.py
|
||||
"""
|
||||
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from dlp_scanner.models import (
|
||||
Finding,
|
||||
Location,
|
||||
ScanResult,
|
||||
)
|
||||
from dlp_scanner.reporters.sarif import (
|
||||
SarifReporter,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def result_with_findings() -> ScanResult:
|
||||
"""
|
||||
Provide a ScanResult with test findings
|
||||
"""
|
||||
result = ScanResult(targets_scanned = 2)
|
||||
result.findings = [
|
||||
Finding(
|
||||
rule_id = "PII_SSN",
|
||||
rule_name = "US Social Security Number",
|
||||
severity = "critical",
|
||||
confidence = 0.95,
|
||||
location = Location(
|
||||
source_type = "file",
|
||||
uri = "data/employees.csv",
|
||||
line = 10,
|
||||
column = 5,
|
||||
),
|
||||
redacted_snippet = "***-**-6789",
|
||||
compliance_frameworks = [
|
||||
"HIPAA",
|
||||
"CCPA",
|
||||
],
|
||||
remediation = "Encrypt SSN data",
|
||||
),
|
||||
Finding(
|
||||
rule_id = "CRED_AWS_ACCESS_KEY",
|
||||
rule_name = "AWS Access Key",
|
||||
severity = "high",
|
||||
confidence = 0.85,
|
||||
location = Location(
|
||||
source_type = "database",
|
||||
uri = "postgresql://host/db",
|
||||
table_name = "config",
|
||||
),
|
||||
redacted_snippet = "AKIA****",
|
||||
compliance_frameworks = [],
|
||||
remediation = "Rotate credentials",
|
||||
),
|
||||
]
|
||||
return result
|
||||
|
||||
|
||||
class TestSarifReporter:
|
||||
def test_generates_valid_json(
|
||||
self,
|
||||
result_with_findings: ScanResult
|
||||
) -> None:
|
||||
reporter = SarifReporter()
|
||||
output = reporter.generate(result_with_findings)
|
||||
data = json.loads(output)
|
||||
assert isinstance(data, dict)
|
||||
|
||||
def test_has_sarif_version(
|
||||
self,
|
||||
result_with_findings: ScanResult
|
||||
) -> None:
|
||||
reporter = SarifReporter()
|
||||
output = reporter.generate(result_with_findings)
|
||||
data = json.loads(output)
|
||||
assert data["version"] == "2.1.0"
|
||||
assert "$schema" in data
|
||||
|
||||
def test_has_tool_driver(
|
||||
self,
|
||||
result_with_findings: ScanResult
|
||||
) -> None:
|
||||
reporter = SarifReporter()
|
||||
output = reporter.generate(result_with_findings)
|
||||
data = json.loads(output)
|
||||
driver = data["runs"][0]["tool"]["driver"]
|
||||
assert driver["name"] == "dlp-scanner"
|
||||
assert driver["version"] == "0.1.0"
|
||||
|
||||
def test_rules_collected(
|
||||
self,
|
||||
result_with_findings: ScanResult
|
||||
) -> None:
|
||||
reporter = SarifReporter()
|
||||
output = reporter.generate(result_with_findings)
|
||||
data = json.loads(output)
|
||||
rules = (data["runs"][0]["tool"]["driver"]["rules"])
|
||||
assert len(rules) == 2
|
||||
rule_ids = {r["id"] for r in rules}
|
||||
assert "PII_SSN" in rule_ids
|
||||
assert "CRED_AWS_ACCESS_KEY" in rule_ids
|
||||
|
||||
def test_results_match_findings(
|
||||
self,
|
||||
result_with_findings: ScanResult
|
||||
) -> None:
|
||||
reporter = SarifReporter()
|
||||
output = reporter.generate(result_with_findings)
|
||||
data = json.loads(output)
|
||||
results = data["runs"][0]["results"]
|
||||
assert len(results) == 2
|
||||
|
||||
def test_severity_mapped_to_level(
|
||||
self,
|
||||
result_with_findings: ScanResult
|
||||
) -> None:
|
||||
reporter = SarifReporter()
|
||||
output = reporter.generate(result_with_findings)
|
||||
data = json.loads(output)
|
||||
results = data["runs"][0]["results"]
|
||||
assert results[0]["level"] == "error"
|
||||
assert results[1]["level"] == "error"
|
||||
|
||||
def test_location_has_artifact(
|
||||
self,
|
||||
result_with_findings: ScanResult
|
||||
) -> None:
|
||||
reporter = SarifReporter()
|
||||
output = reporter.generate(result_with_findings)
|
||||
data = json.loads(output)
|
||||
loc = data["runs"][0]["results"][0]["locations"][0]
|
||||
physical = loc["physicalLocation"]
|
||||
assert (
|
||||
physical["artifactLocation"]["uri"] == "data/employees.csv"
|
||||
)
|
||||
assert physical["region"]["startLine"] == 10
|
||||
assert (physical["region"]["startColumn"] == 5)
|
||||
|
||||
def test_database_finding_has_logical_location(
|
||||
self,
|
||||
result_with_findings: ScanResult
|
||||
) -> None:
|
||||
reporter = SarifReporter()
|
||||
output = reporter.generate(result_with_findings)
|
||||
data = json.loads(output)
|
||||
loc = data["runs"][0]["results"][1]["locations"][0]
|
||||
logical = loc["logicalLocations"]
|
||||
assert logical[0]["name"] == "config"
|
||||
assert logical[0]["kind"] == "table"
|
||||
|
||||
def test_properties_has_confidence(
|
||||
self,
|
||||
result_with_findings: ScanResult
|
||||
) -> None:
|
||||
reporter = SarifReporter()
|
||||
output = reporter.generate(result_with_findings)
|
||||
data = json.loads(output)
|
||||
props = data["runs"][0]["results"][0]["properties"]
|
||||
assert props["confidence"] == 0.95
|
||||
assert props["redactedSnippet"]
|
||||
assert "HIPAA" in (props["complianceFrameworks"])
|
||||
|
||||
def test_empty_result(self) -> None:
|
||||
reporter = SarifReporter()
|
||||
result = ScanResult()
|
||||
output = reporter.generate(result)
|
||||
data = json.loads(output)
|
||||
assert len(data["runs"][0]["results"]) == 0
|
||||
assert (len(data["runs"][0]["tool"]["driver"]["rules"]) == 0)
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
__init__.py
|
||||
"""
|
||||
|
|
@ -0,0 +1,278 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
test_db_scanner.py
|
||||
"""
|
||||
|
||||
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from dlp_scanner.config import ScanConfig
|
||||
from dlp_scanner.detectors.registry import DetectorRegistry
|
||||
from dlp_scanner.scanners.db_scanner import (
|
||||
DatabaseScanner,
|
||||
_extract_mongo_strings,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sqlite_db_with_pii(temp_dir: Path) -> str:
|
||||
"""
|
||||
Provide a SQLite database containing PII test data
|
||||
"""
|
||||
db_path = temp_dir / "test.db"
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
conn.execute(
|
||||
"CREATE TABLE employees ("
|
||||
"id INTEGER PRIMARY KEY, "
|
||||
"name TEXT, "
|
||||
"ssn TEXT, "
|
||||
"email TEXT, "
|
||||
"salary REAL)"
|
||||
)
|
||||
conn.execute(
|
||||
"INSERT INTO employees "
|
||||
"(name, ssn, email, salary) "
|
||||
"VALUES (?, ?, ?, ?)",
|
||||
(
|
||||
"John Doe",
|
||||
"456-78-9012",
|
||||
"john@example.com",
|
||||
75000.0,
|
||||
),
|
||||
)
|
||||
conn.execute(
|
||||
"INSERT INTO employees "
|
||||
"(name, ssn, email, salary) "
|
||||
"VALUES (?, ?, ?, ?)",
|
||||
(
|
||||
"Jane Smith",
|
||||
"234-56-7890",
|
||||
"jane@example.com",
|
||||
85000.0,
|
||||
),
|
||||
)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
return f"sqlite:///{db_path}"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sqlite_db_empty(temp_dir: Path) -> str:
|
||||
"""
|
||||
Provide a SQLite database with an empty table
|
||||
"""
|
||||
db_path = temp_dir / "empty.db"
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
conn.execute(
|
||||
"CREATE TABLE logs ("
|
||||
"id INTEGER PRIMARY KEY, "
|
||||
"message TEXT)"
|
||||
)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
return f"sqlite:///{db_path}"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def db_scanner() -> DatabaseScanner:
|
||||
"""
|
||||
Provide a default DatabaseScanner instance
|
||||
"""
|
||||
config = ScanConfig()
|
||||
registry = DetectorRegistry()
|
||||
return DatabaseScanner(config = config, registry = registry)
|
||||
|
||||
|
||||
class TestDatabaseScanner:
|
||||
def test_sqlite_scan_finds_pii(
|
||||
self,
|
||||
db_scanner: DatabaseScanner,
|
||||
sqlite_db_with_pii: str,
|
||||
) -> None:
|
||||
result = db_scanner.scan(sqlite_db_with_pii)
|
||||
assert result.targets_scanned > 0
|
||||
assert len(result.findings) > 0
|
||||
|
||||
def test_sqlite_scan_finds_ssn(
|
||||
self,
|
||||
db_scanner: DatabaseScanner,
|
||||
sqlite_db_with_pii: str,
|
||||
) -> None:
|
||||
result = db_scanner.scan(sqlite_db_with_pii)
|
||||
ssn_findings = [
|
||||
f for f in result.findings if f.rule_id == "PII_SSN"
|
||||
]
|
||||
assert len(ssn_findings) > 0
|
||||
|
||||
def test_sqlite_scan_empty_table(
|
||||
self,
|
||||
db_scanner: DatabaseScanner,
|
||||
sqlite_db_empty: str,
|
||||
) -> None:
|
||||
result = db_scanner.scan(sqlite_db_empty)
|
||||
assert result.targets_scanned > 0
|
||||
assert len(result.findings) == 0
|
||||
|
||||
def test_findings_have_database_source(
|
||||
self,
|
||||
db_scanner: DatabaseScanner,
|
||||
sqlite_db_with_pii: str,
|
||||
) -> None:
|
||||
result = db_scanner.scan(sqlite_db_with_pii)
|
||||
for finding in result.findings:
|
||||
assert (finding.location.source_type == "database")
|
||||
|
||||
def test_findings_have_table_name(
|
||||
self,
|
||||
db_scanner: DatabaseScanner,
|
||||
sqlite_db_with_pii: str,
|
||||
) -> None:
|
||||
result = db_scanner.scan(sqlite_db_with_pii)
|
||||
for finding in result.findings:
|
||||
assert (finding.location.table_name == "employees")
|
||||
|
||||
def test_unsupported_scheme_errors(
|
||||
self,
|
||||
db_scanner: DatabaseScanner,
|
||||
) -> None:
|
||||
result = db_scanner.scan("ftp://localhost/db")
|
||||
assert len(result.errors) > 0
|
||||
|
||||
def test_completed_at_is_set(
|
||||
self,
|
||||
db_scanner: DatabaseScanner,
|
||||
sqlite_db_with_pii: str,
|
||||
) -> None:
|
||||
result = db_scanner.scan(sqlite_db_with_pii)
|
||||
assert result.scan_completed_at is not None
|
||||
|
||||
def test_findings_have_remediation(
|
||||
self,
|
||||
db_scanner: DatabaseScanner,
|
||||
sqlite_db_with_pii: str,
|
||||
) -> None:
|
||||
result = db_scanner.scan(sqlite_db_with_pii)
|
||||
for finding in result.findings:
|
||||
assert finding.remediation
|
||||
|
||||
def test_table_exclude_filter(
|
||||
self,
|
||||
temp_dir: Path,
|
||||
) -> None:
|
||||
db_path = temp_dir / "filter.db"
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
conn.execute("CREATE TABLE users "
|
||||
"(id INTEGER, ssn TEXT)")
|
||||
conn.execute("INSERT INTO users "
|
||||
"VALUES (1, '123-45-6789')")
|
||||
conn.execute("CREATE TABLE audit_log "
|
||||
"(id INTEGER, note TEXT)")
|
||||
conn.execute("INSERT INTO audit_log "
|
||||
"VALUES (1, '987-65-4321')")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
config = ScanConfig()
|
||||
config.database.exclude_tables = ["audit_log"]
|
||||
registry = DetectorRegistry()
|
||||
scanner = DatabaseScanner(config = config, registry = registry)
|
||||
|
||||
result = scanner.scan(f"sqlite:///{db_path}")
|
||||
assert result.targets_scanned == 1
|
||||
|
||||
def test_table_include_filter(
|
||||
self,
|
||||
temp_dir: Path,
|
||||
) -> None:
|
||||
db_path = temp_dir / "include.db"
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
conn.execute("CREATE TABLE users "
|
||||
"(id INTEGER, ssn TEXT)")
|
||||
conn.execute("INSERT INTO users "
|
||||
"VALUES (1, '123-45-6789')")
|
||||
conn.execute("CREATE TABLE logs "
|
||||
"(id INTEGER, msg TEXT)")
|
||||
conn.execute("INSERT INTO logs "
|
||||
"VALUES (1, '987-65-4321')")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
config = ScanConfig()
|
||||
config.database.include_tables = ["users"]
|
||||
registry = DetectorRegistry()
|
||||
scanner = DatabaseScanner(config = config, registry = registry)
|
||||
|
||||
result = scanner.scan(f"sqlite:///{db_path}")
|
||||
assert result.targets_scanned == 1
|
||||
|
||||
|
||||
class TestExtractMongoStrings:
|
||||
def test_simple_strings(self) -> None:
|
||||
doc: dict[str,
|
||||
Any] = {
|
||||
"name": "John",
|
||||
"email": "john@test.com",
|
||||
}
|
||||
parts: list[str] = []
|
||||
_extract_mongo_strings(doc, parts)
|
||||
assert len(parts) == 2
|
||||
|
||||
def test_nested_doc(self) -> None:
|
||||
doc: dict[str,
|
||||
Any] = {
|
||||
"user": {
|
||||
"name": "Jane",
|
||||
"ssn": "123-45-6789",
|
||||
}
|
||||
}
|
||||
parts: list[str] = []
|
||||
_extract_mongo_strings(doc, parts)
|
||||
assert any("user.name" in p for p in parts)
|
||||
assert any("user.ssn" in p for p in parts)
|
||||
|
||||
def test_skips_id_field(self) -> None:
|
||||
doc: dict[str,
|
||||
Any] = {
|
||||
"_id": "abc123",
|
||||
"name": "Test",
|
||||
}
|
||||
parts: list[str] = []
|
||||
_extract_mongo_strings(doc, parts)
|
||||
assert len(parts) == 1
|
||||
assert "name" in parts[0]
|
||||
|
||||
def test_list_values(self) -> None:
|
||||
doc: dict[str, Any] = {"emails": ["a@b.com", "c@d.com"]}
|
||||
parts: list[str] = []
|
||||
_extract_mongo_strings(doc, parts)
|
||||
assert len(parts) == 2
|
||||
|
||||
def test_empty_strings_skipped(self) -> None:
|
||||
doc: dict[str,
|
||||
Any] = {
|
||||
"name": "",
|
||||
"bio": " ",
|
||||
}
|
||||
parts: list[str] = []
|
||||
_extract_mongo_strings(doc, parts)
|
||||
assert len(parts) == 0
|
||||
|
||||
def test_nested_list_of_dicts(self) -> None:
|
||||
doc: dict[str,
|
||||
Any] = {
|
||||
"records": [
|
||||
{
|
||||
"value": "secret"
|
||||
},
|
||||
{
|
||||
"value": "data"
|
||||
},
|
||||
]
|
||||
}
|
||||
parts: list[str] = []
|
||||
_extract_mongo_strings(doc, parts)
|
||||
assert len(parts) == 2
|
||||
|
|
@ -0,0 +1,210 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
test_file_scanner.py
|
||||
"""
|
||||
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from dlp_scanner.config import ScanConfig
|
||||
from dlp_scanner.detectors.registry import DetectorRegistry
|
||||
from dlp_scanner.scanners.file_scanner import (
|
||||
FileScanner,
|
||||
_build_extension_map,
|
||||
_get_full_suffix,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def file_scanner() -> FileScanner:
|
||||
"""
|
||||
Provide a default FileScanner instance
|
||||
"""
|
||||
config = ScanConfig()
|
||||
registry = DetectorRegistry()
|
||||
return FileScanner(config = config, registry = registry)
|
||||
|
||||
|
||||
class TestFileScanner:
|
||||
def test_scan_directory_finds_pii(
|
||||
self,
|
||||
file_scanner: FileScanner,
|
||||
temp_dir_with_pii: Path,
|
||||
) -> None:
|
||||
result = file_scanner.scan(str(temp_dir_with_pii))
|
||||
assert result.targets_scanned > 0
|
||||
assert len(result.findings) > 0
|
||||
|
||||
def test_scan_single_file(
|
||||
self,
|
||||
file_scanner: FileScanner,
|
||||
temp_dir_with_pii: Path,
|
||||
) -> None:
|
||||
csv_path = temp_dir_with_pii / "employees.csv"
|
||||
result = file_scanner.scan(str(csv_path))
|
||||
assert result.targets_scanned == 1
|
||||
assert len(result.findings) > 0
|
||||
|
||||
def test_scan_clean_file_no_findings(
|
||||
self,
|
||||
file_scanner: FileScanner,
|
||||
temp_dir_with_pii: Path,
|
||||
) -> None:
|
||||
txt_path = temp_dir_with_pii / "clean.txt"
|
||||
result = file_scanner.scan(str(txt_path))
|
||||
assert result.targets_scanned == 1
|
||||
assert len(result.findings) == 0
|
||||
|
||||
def test_scan_nonexistent_target(
|
||||
self,
|
||||
file_scanner: FileScanner,
|
||||
) -> None:
|
||||
result = file_scanner.scan("/nonexistent/path")
|
||||
assert len(result.errors) > 0
|
||||
|
||||
def test_scan_empty_directory(
|
||||
self,
|
||||
file_scanner: FileScanner,
|
||||
temp_dir: Path,
|
||||
) -> None:
|
||||
result = file_scanner.scan(str(temp_dir))
|
||||
assert result.targets_scanned == 0
|
||||
assert len(result.findings) == 0
|
||||
|
||||
def test_scan_respects_exclude_patterns(
|
||||
self,
|
||||
temp_dir: Path,
|
||||
) -> None:
|
||||
secret = temp_dir / "secret.log"
|
||||
secret.write_text("SSN: 123-45-6789")
|
||||
|
||||
config = ScanConfig()
|
||||
config.file.exclude_patterns = ["*.log"]
|
||||
registry = DetectorRegistry()
|
||||
scanner = FileScanner(config = config, registry = registry)
|
||||
|
||||
result = scanner.scan(str(temp_dir))
|
||||
assert result.targets_scanned == 0
|
||||
|
||||
def test_scan_respects_max_file_size(
|
||||
self,
|
||||
temp_dir: Path,
|
||||
) -> None:
|
||||
large = temp_dir / "large.txt"
|
||||
large.write_text("SSN: 123-45-6789\n" * 100)
|
||||
|
||||
config = ScanConfig()
|
||||
config.file.max_file_size_mb = 0
|
||||
registry = DetectorRegistry()
|
||||
scanner = FileScanner(config = config, registry = registry)
|
||||
|
||||
result = scanner.scan(str(temp_dir))
|
||||
assert result.targets_scanned == 0
|
||||
|
||||
def test_scan_completed_at_is_set(
|
||||
self,
|
||||
file_scanner: FileScanner,
|
||||
temp_dir: Path,
|
||||
) -> None:
|
||||
result = file_scanner.scan(str(temp_dir))
|
||||
assert result.scan_completed_at is not None
|
||||
|
||||
def test_findings_have_compliance_frameworks(
|
||||
self,
|
||||
file_scanner: FileScanner,
|
||||
temp_dir_with_pii: Path,
|
||||
) -> None:
|
||||
result = file_scanner.scan(str(temp_dir_with_pii))
|
||||
ssn_findings = [
|
||||
f for f in result.findings if f.rule_id == "PII_SSN"
|
||||
]
|
||||
for finding in ssn_findings:
|
||||
assert len(finding.compliance_frameworks) > 0
|
||||
|
||||
def test_findings_have_redacted_snippets(
|
||||
self,
|
||||
file_scanner: FileScanner,
|
||||
temp_dir_with_pii: Path,
|
||||
) -> None:
|
||||
result = file_scanner.scan(str(temp_dir_with_pii))
|
||||
for finding in result.findings:
|
||||
assert finding.redacted_snippet
|
||||
|
||||
def test_findings_have_severity(
|
||||
self,
|
||||
file_scanner: FileScanner,
|
||||
temp_dir_with_pii: Path,
|
||||
) -> None:
|
||||
result = file_scanner.scan(str(temp_dir_with_pii))
|
||||
valid_severities = {
|
||||
"critical",
|
||||
"high",
|
||||
"medium",
|
||||
"low",
|
||||
}
|
||||
for finding in result.findings:
|
||||
assert finding.severity in valid_severities
|
||||
|
||||
def test_scan_json_finds_api_key(
|
||||
self,
|
||||
file_scanner: FileScanner,
|
||||
temp_dir_with_pii: Path,
|
||||
) -> None:
|
||||
result = file_scanner.scan(str(temp_dir_with_pii))
|
||||
cred_findings = [
|
||||
f for f in result.findings if f.rule_id.startswith("CRED_")
|
||||
]
|
||||
assert len(cred_findings) > 0
|
||||
|
||||
|
||||
class TestExtensionMap:
|
||||
def test_has_common_text_types(self) -> None:
|
||||
ext_map = _build_extension_map()
|
||||
assert ".txt" in ext_map
|
||||
assert ".csv" in ext_map
|
||||
assert ".json" in ext_map
|
||||
assert ".xml" in ext_map
|
||||
assert ".yaml" in ext_map
|
||||
|
||||
def test_has_office_types(self) -> None:
|
||||
ext_map = _build_extension_map()
|
||||
assert ".pdf" in ext_map
|
||||
assert ".docx" in ext_map
|
||||
assert ".xlsx" in ext_map
|
||||
assert ".xls" in ext_map
|
||||
|
||||
def test_has_archive_types(self) -> None:
|
||||
ext_map = _build_extension_map()
|
||||
assert ".zip" in ext_map
|
||||
assert ".tar" in ext_map
|
||||
assert ".tar.gz" in ext_map
|
||||
|
||||
def test_has_email_types(self) -> None:
|
||||
ext_map = _build_extension_map()
|
||||
assert ".eml" in ext_map
|
||||
assert ".msg" in ext_map
|
||||
|
||||
|
||||
class TestGetFullSuffix:
|
||||
def test_simple_extension(self) -> None:
|
||||
assert _get_full_suffix(Path("f.txt")) == ".txt"
|
||||
|
||||
def test_tar_gz(self) -> None:
|
||||
path = Path("archive.tar.gz")
|
||||
assert _get_full_suffix(path) == ".tar.gz"
|
||||
|
||||
def test_tar_bz2(self) -> None:
|
||||
path = Path("archive.tar.bz2")
|
||||
assert _get_full_suffix(path) == ".tar.bz2"
|
||||
|
||||
def test_uppercase_normalized(self) -> None:
|
||||
assert _get_full_suffix(Path("F.TXT")) == ".txt"
|
||||
|
||||
def test_no_extension(self) -> None:
|
||||
assert _get_full_suffix(Path("Makefile")) == ""
|
||||
|
||||
def test_dotfile(self) -> None:
|
||||
result = _get_full_suffix(Path(".gitignore"))
|
||||
assert result == ""
|
||||
File diff suppressed because it is too large
Load Diff
Loading…
Reference in New Issue