feat: Complete Go secrets scanner - Portia
This commit is contained in:
CarterPerez-dev
parent
c6c844db47
commit
294169a222
|
|
@ -0,0 +1,7 @@
|
|||
# ©AngelaMos | 2026
|
||||
# .gitguardian.yml
|
||||
|
||||
version: 2
|
||||
|
||||
ignore-paths:
|
||||
- PROJECTS/intermediate/secrets-scanner/
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
# ©AngelaMos | 2026
|
||||
# secret_scanning.yml
|
||||
|
||||
paths-ignore:
|
||||
- 'PROJECTS/intermediate/secrets-scanner/**'
|
||||
|
|
@ -86,6 +86,9 @@ jobs:
|
|||
- name: docker-security-audit
|
||||
type: go
|
||||
path: PROJECTS/intermediate/docker-security-audit
|
||||
- name: secrets-scanner
|
||||
type: go
|
||||
path: PROJECTS/intermediate/secrets-scanner
|
||||
|
||||
defaults:
|
||||
run:
|
||||
|
|
|
|||
|
|
@ -104,6 +104,14 @@ repos:
|
|||
types: [go]
|
||||
pass_filenames: false
|
||||
|
||||
- id: golangci-lint-secrets-scanner
|
||||
name: golangci-lint (secrets-scanner)
|
||||
entry: bash -c 'cd PROJECTS/intermediate/secrets-scanner && golangci-lint run --fix'
|
||||
language: system
|
||||
files: ^PROJECTS/intermediate/secrets-scanner/
|
||||
types: [go]
|
||||
pass_filenames: false
|
||||
|
||||
# Biome Frontend Checks
|
||||
- repo: local
|
||||
hooks:
|
||||
|
|
|
|||
|
|
@ -0,0 +1,15 @@
|
|||
# ©AngelaMos | 2026
|
||||
# .gitignore
|
||||
|
||||
.venv
|
||||
venv/
|
||||
__pycache__
|
||||
*.pyc
|
||||
.mypy_cache
|
||||
.pytest_cache
|
||||
.ruff_cache
|
||||
htmlcov
|
||||
.coverage
|
||||
dist
|
||||
*.egg-info
|
||||
.hypothesis
|
||||
|
|
@ -31,9 +31,8 @@ lint-fix:
|
|||
golangci-lint run --timeout=5m --fix
|
||||
|
||||
[group('lint')]
|
||||
fmt:
|
||||
gofumpt -w .
|
||||
golines -w --max-len=80 .
|
||||
format:
|
||||
golangci-lint fmt
|
||||
|
||||
[group('lint')]
|
||||
tidy:
|
||||
|
|
|
|||
|
|
@ -43,8 +43,5 @@ build/
|
|||
.coverage
|
||||
htmlcov/
|
||||
|
||||
# Docker
|
||||
.dockerignore
|
||||
|
||||
# OS
|
||||
Thumbs.db
|
||||
|
|
|
|||
|
|
@ -0,0 +1,11 @@
|
|||
# AngelaMos | 2026
|
||||
# .gitignore
|
||||
|
||||
bin/
|
||||
*.exe
|
||||
*.test
|
||||
*.out
|
||||
coverage.out
|
||||
coverage.html
|
||||
.portia-cache/
|
||||
.portia.toml
|
||||
|
|
@ -0,0 +1,120 @@
|
|||
# AngelaMos | 2026
|
||||
# .golangci.yml
|
||||
|
||||
version: "2"
|
||||
|
||||
linters:
|
||||
default: none
|
||||
enable:
|
||||
- errcheck
|
||||
- govet
|
||||
- gosec
|
||||
- bodyclose
|
||||
- nilerr
|
||||
- errorlint
|
||||
- exhaustive
|
||||
- gocritic
|
||||
- funlen
|
||||
- gocognit
|
||||
- dupl
|
||||
- goconst
|
||||
- ineffassign
|
||||
- unused
|
||||
- unconvert
|
||||
- unparam
|
||||
- testifylint
|
||||
- fatcontext
|
||||
|
||||
exclusions:
|
||||
rules:
|
||||
- path: _test\.go
|
||||
linters:
|
||||
- funlen
|
||||
- dupl
|
||||
- goconst
|
||||
- gosec
|
||||
- path: builtin\.go
|
||||
linters:
|
||||
- funlen
|
||||
- dupl
|
||||
- goconst
|
||||
paths:
|
||||
- vendor
|
||||
- testdata
|
||||
max-same-issues: 50
|
||||
|
||||
settings:
|
||||
errcheck:
|
||||
check-type-assertions: true
|
||||
check-blank: true
|
||||
|
||||
funlen:
|
||||
lines: 120
|
||||
statements: 60
|
||||
|
||||
gocognit:
|
||||
min-complexity: 25
|
||||
|
||||
govet:
|
||||
enable-all: true
|
||||
disable:
|
||||
- fieldalignment
|
||||
|
||||
revive:
|
||||
rules:
|
||||
- name: blank-imports
|
||||
- name: context-as-argument
|
||||
- name: context-keys-type
|
||||
- name: error-return
|
||||
- name: error-strings
|
||||
- name: error-naming
|
||||
- name: exported
|
||||
- name: increment-decrement
|
||||
- name: var-declaration
|
||||
- name: package-comments
|
||||
disabled: true
|
||||
- name: range
|
||||
- name: receiver-naming
|
||||
- name: time-naming
|
||||
- name: unexported-return
|
||||
- name: indent-error-flow
|
||||
- name: errorf
|
||||
- name: empty-block
|
||||
- name: superfluous-else
|
||||
- name: unreachable-code
|
||||
|
||||
staticcheck:
|
||||
checks:
|
||||
- all
|
||||
|
||||
gosec:
|
||||
excludes:
|
||||
- G104
|
||||
- G401
|
||||
- G505
|
||||
|
||||
sloglint:
|
||||
no-mixed-args: true
|
||||
kv-only: true
|
||||
context: all
|
||||
|
||||
formatters:
|
||||
enable:
|
||||
- gci
|
||||
- gofumpt
|
||||
- golines
|
||||
settings:
|
||||
golines:
|
||||
max-len: 80
|
||||
reformat-tags: true
|
||||
goimports:
|
||||
local-prefixes:
|
||||
- github.com/CarterPerez-dev/portia
|
||||
gci:
|
||||
sections:
|
||||
- standard
|
||||
- default
|
||||
- prefix(github.com/CarterPerez-dev)
|
||||
custom-order: true
|
||||
gofumpt:
|
||||
extra-rules: true
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
# ©AngelaMos | 2026
|
||||
# .portia.toml
|
||||
|
||||
[rules]
|
||||
# disable = ["generic-password"]
|
||||
|
||||
[scan]
|
||||
max_file_size = 1048576
|
||||
# excludes = ["*.test.js", "fixtures/"]
|
||||
# depth = 0
|
||||
# since = ""
|
||||
|
||||
[output]
|
||||
format = "terminal"
|
||||
verbose = false
|
||||
no_color = false
|
||||
|
||||
[hibp]
|
||||
enabled = false
|
||||
|
||||
[allowlist]
|
||||
# paths = ["test/fixtures/"]
|
||||
# values = ["my_test_value"]
|
||||
# stopwords = ["mycompany"]
|
||||
|
|
@ -0,0 +1,143 @@
|
|||
# =============================================================================
|
||||
# AngelaMos | 2026
|
||||
# Justfile
|
||||
# =============================================================================
|
||||
# portia — Secrets scanner for codebases and git repositories
|
||||
# =============================================================================
|
||||
|
||||
set export
|
||||
set shell := ["bash", "-uc"]
|
||||
|
||||
project := file_name(justfile_directory())
|
||||
version := `git describe --tags --always 2>/dev/null || echo "dev"`
|
||||
|
||||
# =============================================================================
|
||||
# Default
|
||||
# =============================================================================
|
||||
|
||||
default:
|
||||
@just --list --unsorted
|
||||
|
||||
# =============================================================================
|
||||
# Linting and Formatting
|
||||
# =============================================================================
|
||||
|
||||
[group('lint')]
|
||||
lint *ARGS:
|
||||
golangci-lint run --timeout=5m {{ARGS}}
|
||||
|
||||
[group('lint')]
|
||||
lint-fix:
|
||||
golangci-lint run --timeout=5m --fix
|
||||
|
||||
[group('lint')]
|
||||
format:
|
||||
golangci-lint fmt
|
||||
|
||||
[group('lint')]
|
||||
tidy:
|
||||
go mod tidy
|
||||
|
||||
[group('lint')]
|
||||
vet:
|
||||
go vet ./...
|
||||
|
||||
# =============================================================================
|
||||
# Testing
|
||||
# =============================================================================
|
||||
|
||||
[group('test')]
|
||||
test *ARGS:
|
||||
go test -race ./... {{ARGS}}
|
||||
|
||||
[group('test')]
|
||||
test-v *ARGS:
|
||||
go test -race -v ./... {{ARGS}}
|
||||
|
||||
[group('test')]
|
||||
cover:
|
||||
go test -race -cover ./...
|
||||
|
||||
[group('test')]
|
||||
cover-html:
|
||||
go test -race -coverprofile=coverage.out ./...
|
||||
go tool cover -html=coverage.out -o coverage.html
|
||||
@echo "Coverage report: coverage.html"
|
||||
|
||||
# =============================================================================
|
||||
# CI / Quality
|
||||
# =============================================================================
|
||||
|
||||
[group('ci')]
|
||||
ci: lint test
|
||||
@echo "All checks passed."
|
||||
|
||||
[group('ci')]
|
||||
check: lint vet
|
||||
|
||||
# =============================================================================
|
||||
# Development
|
||||
# =============================================================================
|
||||
|
||||
[group('dev')]
|
||||
run *ARGS:
|
||||
go run ./cmd/portia {{ARGS}}
|
||||
|
||||
[group('dev')]
|
||||
dev-scan:
|
||||
go run ./cmd/portia scan testdata/
|
||||
|
||||
[group('dev')]
|
||||
dev-git:
|
||||
go run ./cmd/portia git .
|
||||
|
||||
[group('dev')]
|
||||
dev-json:
|
||||
go run ./cmd/portia scan --format json testdata/
|
||||
|
||||
[group('dev')]
|
||||
dev-sarif:
|
||||
go run ./cmd/portia scan --format sarif testdata/
|
||||
|
||||
[group('dev')]
|
||||
dev-rules:
|
||||
go run ./cmd/portia config rules
|
||||
|
||||
# =============================================================================
|
||||
# Build (Production)
|
||||
# =============================================================================
|
||||
|
||||
[group('prod')]
|
||||
build:
|
||||
go build -ldflags="-s -w" -o bin/portia ./cmd/portia
|
||||
@echo "Built: bin/portia ($(du -h bin/portia | cut -f1))"
|
||||
|
||||
[group('prod')]
|
||||
build-debug:
|
||||
go build -o bin/portia ./cmd/portia
|
||||
|
||||
[group('prod')]
|
||||
install:
|
||||
go install ./cmd/portia
|
||||
|
||||
# =============================================================================
|
||||
# Utilities
|
||||
# =============================================================================
|
||||
|
||||
[group('util')]
|
||||
info:
|
||||
@echo "Project: {{project}}"
|
||||
@echo "Version: {{version}}"
|
||||
@echo "Go: $(go version | cut -d' ' -f3)"
|
||||
@echo "OS: {{os()}} ({{arch()}})"
|
||||
@echo "Module: $(head -1 go.mod | cut -d' ' -f2)"
|
||||
|
||||
[group('util')]
|
||||
update:
|
||||
go get -u ./...
|
||||
go mod tidy
|
||||
|
||||
[group('util')]
|
||||
clean:
|
||||
-rm -rf bin/ coverage.out coverage.html
|
||||
@echo "Cleaned build artifacts."
|
||||
|
|
@ -0,0 +1,661 @@
|
|||
GNU AFFERO GENERAL PUBLIC LICENSE
|
||||
Version 3, 19 November 2007
|
||||
|
||||
Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
|
||||
Everyone is permitted to copy and distribute verbatim copies
|
||||
of this license document, but changing it is not allowed.
|
||||
|
||||
Preamble
|
||||
|
||||
The GNU Affero General Public License is a free, copyleft license for
|
||||
software and other kinds of works, specifically designed to ensure
|
||||
cooperation with the community in the case of network server software.
|
||||
|
||||
The licenses for most software and other practical works are designed
|
||||
to take away your freedom to share and change the works. By contrast,
|
||||
our General Public Licenses are intended to guarantee your freedom to
|
||||
share and change all versions of a program--to make sure it remains free
|
||||
software for all its users.
|
||||
|
||||
When we speak of free software, we are referring to freedom, not
|
||||
price. Our General Public Licenses are designed to make sure that you
|
||||
have the freedom to distribute copies of free software (and charge for
|
||||
them if you wish), that you receive source code or can get it if you
|
||||
want it, that you can change the software or use pieces of it in new
|
||||
free programs, and that you know you can do these things.
|
||||
|
||||
Developers that use our General Public Licenses protect your rights
|
||||
with two steps: (1) assert copyright on the software, and (2) offer
|
||||
you this License which gives you legal permission to copy, distribute
|
||||
and/or modify the software.
|
||||
|
||||
A secondary benefit of defending all users' freedom is that
|
||||
improvements made in alternate versions of the program, if they
|
||||
receive widespread use, become available for other developers to
|
||||
incorporate. Many developers of free software are heartened and
|
||||
encouraged by the resulting cooperation. However, in the case of
|
||||
software used on network servers, this result may fail to come about.
|
||||
The GNU General Public License permits making a modified version and
|
||||
letting the public access it on a server without ever releasing its
|
||||
source code to the public.
|
||||
|
||||
The GNU Affero General Public License is designed specifically to
|
||||
ensure that, in such cases, the modified source code becomes available
|
||||
to the community. It requires the operator of a network server to
|
||||
provide the source code of the modified version running there to the
|
||||
users of that server. Therefore, public use of a modified version, on
|
||||
a publicly accessible server, gives the public access to the source
|
||||
code of the modified version.
|
||||
|
||||
An older license, called the Affero General Public License and
|
||||
published by Affero, was designed to accomplish similar goals. This is
|
||||
a different license, not a version of the Affero GPL, but Affero has
|
||||
released a new version of the Affero GPL which permits relicensing under
|
||||
this license.
|
||||
|
||||
The precise terms and conditions for copying, distribution and
|
||||
modification follow.
|
||||
|
||||
TERMS AND CONDITIONS
|
||||
|
||||
0. Definitions.
|
||||
|
||||
"This License" refers to version 3 of the GNU Affero General Public License.
|
||||
|
||||
"Copyright" also means copyright-like laws that apply to other kinds of
|
||||
works, such as semiconductor masks.
|
||||
|
||||
"The Program" refers to any copyrightable work licensed under this
|
||||
License. Each licensee is addressed as "you". "Licensees" and
|
||||
"recipients" may be individuals or organizations.
|
||||
|
||||
To "modify" a work means to copy from or adapt all or part of the work
|
||||
in a fashion requiring copyright permission, other than the making of an
|
||||
exact copy. The resulting work is called a "modified version" of the
|
||||
earlier work or a work "based on" the earlier work.
|
||||
|
||||
A "covered work" means either the unmodified Program or a work based
|
||||
on the Program.
|
||||
|
||||
To "propagate" a work means to do anything with it that, without
|
||||
permission, would make you directly or secondarily liable for
|
||||
infringement under applicable copyright law, except executing it on a
|
||||
computer or modifying a private copy. Propagation includes copying,
|
||||
distribution (with or without modification), making available to the
|
||||
public, and in some countries other activities as well.
|
||||
|
||||
To "convey" a work means any kind of propagation that enables other
|
||||
parties to make or receive copies. Mere interaction with a user through
|
||||
a computer network, with no transfer of a copy, is not conveying.
|
||||
|
||||
An interactive user interface displays "Appropriate Legal Notices"
|
||||
to the extent that it includes a convenient and prominently visible
|
||||
feature that (1) displays an appropriate copyright notice, and (2)
|
||||
tells the user that there is no warranty for the work (except to the
|
||||
extent that warranties are provided), that licensees may convey the
|
||||
work under this License, and how to view a copy of this License. If
|
||||
the interface presents a list of user commands or options, such as a
|
||||
menu, a prominent item in the list meets this criterion.
|
||||
|
||||
1. Source Code.
|
||||
|
||||
The "source code" for a work means the preferred form of the work
|
||||
for making modifications to it. "Object code" means any non-source
|
||||
form of a work.
|
||||
|
||||
A "Standard Interface" means an interface that either is an official
|
||||
standard defined by a recognized standards body, or, in the case of
|
||||
interfaces specified for a particular programming language, one that
|
||||
is widely used among developers working in that language.
|
||||
|
||||
The "System Libraries" of an executable work include anything, other
|
||||
than the work as a whole, that (a) is included in the normal form of
|
||||
packaging a Major Component, but which is not part of that Major
|
||||
Component, and (b) serves only to enable use of the work with that
|
||||
Major Component, or to implement a Standard Interface for which an
|
||||
implementation is available to the public in source code form. A
|
||||
"Major Component", in this context, means a major essential component
|
||||
(kernel, window system, and so on) of the specific operating system
|
||||
(if any) on which the executable work runs, or a compiler used to
|
||||
produce the work, or an object code interpreter used to run it.
|
||||
|
||||
The "Corresponding Source" for a work in object code form means all
|
||||
the source code needed to generate, install, and (for an executable
|
||||
work) run the object code and to modify the work, including scripts to
|
||||
control those activities. However, it does not include the work's
|
||||
System Libraries, or general-purpose tools or generally available free
|
||||
programs which are used unmodified in performing those activities but
|
||||
which are not part of the work. For example, Corresponding Source
|
||||
includes interface definition files associated with source files for
|
||||
the work, and the source code for shared libraries and dynamically
|
||||
linked subprograms that the work is specifically designed to require,
|
||||
such as by intimate data communication or control flow between those
|
||||
subprograms and other parts of the work.
|
||||
|
||||
The Corresponding Source need not include anything that users
|
||||
can regenerate automatically from other parts of the Corresponding
|
||||
Source.
|
||||
|
||||
The Corresponding Source for a work in source code form is that
|
||||
same work.
|
||||
|
||||
2. Basic Permissions.
|
||||
|
||||
All rights granted under this License are granted for the term of
|
||||
copyright on the Program, and are irrevocable provided the stated
|
||||
conditions are met. This License explicitly affirms your unlimited
|
||||
permission to run the unmodified Program. The output from running a
|
||||
covered work is covered by this License only if the output, given its
|
||||
content, constitutes a covered work. This License acknowledges your
|
||||
rights of fair use or other equivalent, as provided by copyright law.
|
||||
|
||||
You may make, run and propagate covered works that you do not
|
||||
convey, without conditions so long as your license otherwise remains
|
||||
in force. You may convey covered works to others for the sole purpose
|
||||
of having them make modifications exclusively for you, or provide you
|
||||
with facilities for running those works, provided that you comply with
|
||||
the terms of this License in conveying all material for which you do
|
||||
not control copyright. Those thus making or running the covered works
|
||||
for you must do so exclusively on your behalf, under your direction
|
||||
and control, on terms that prohibit them from making any copies of
|
||||
your copyrighted material outside their relationship with you.
|
||||
|
||||
Conveying under any other circumstances is permitted solely under
|
||||
the conditions stated below. Sublicensing is not allowed; section 10
|
||||
makes it unnecessary.
|
||||
|
||||
3. Protecting Users' Legal Rights From Anti-Circumvention Law.
|
||||
|
||||
No covered work shall be deemed part of an effective technological
|
||||
measure under any applicable law fulfilling obligations under article
|
||||
11 of the WIPO copyright treaty adopted on 20 December 1996, or
|
||||
similar laws prohibiting or restricting circumvention of such
|
||||
measures.
|
||||
|
||||
When you convey a covered work, you waive any legal power to forbid
|
||||
circumvention of technological measures to the extent such circumvention
|
||||
is effected by exercising rights under this License with respect to
|
||||
the covered work, and you disclaim any intention to limit operation or
|
||||
modification of the work as a means of enforcing, against the work's
|
||||
users, your or third parties' legal rights to forbid circumvention of
|
||||
technological measures.
|
||||
|
||||
4. Conveying Verbatim Copies.
|
||||
|
||||
You may convey verbatim copies of the Program's source code as you
|
||||
receive it, in any medium, provided that you conspicuously and
|
||||
appropriately publish on each copy an appropriate copyright notice;
|
||||
keep intact all notices stating that this License and any
|
||||
non-permissive terms added in accord with section 7 apply to the code;
|
||||
keep intact all notices of the absence of any warranty; and give all
|
||||
recipients a copy of this License along with the Program.
|
||||
|
||||
You may charge any price or no price for each copy that you convey,
|
||||
and you may offer support or warranty protection for a fee.
|
||||
|
||||
5. Conveying Modified Source Versions.
|
||||
|
||||
You may convey a work based on the Program, or the modifications to
|
||||
produce it from the Program, in the form of source code under the
|
||||
terms of section 4, provided that you also meet all of these conditions:
|
||||
|
||||
a) The work must carry prominent notices stating that you modified
|
||||
it, and giving a relevant date.
|
||||
|
||||
b) The work must carry prominent notices stating that it is
|
||||
released under this License and any conditions added under section
|
||||
7. This requirement modifies the requirement in section 4 to
|
||||
"keep intact all notices".
|
||||
|
||||
c) You must license the entire work, as a whole, under this
|
||||
License to anyone who comes into possession of a copy. This
|
||||
License will therefore apply, along with any applicable section 7
|
||||
additional terms, to the whole of the work, and all its parts,
|
||||
regardless of how they are packaged. This License gives no
|
||||
permission to license the work in any other way, but it does not
|
||||
invalidate such permission if you have separately received it.
|
||||
|
||||
d) If the work has interactive user interfaces, each must display
|
||||
Appropriate Legal Notices; however, if the Program has interactive
|
||||
interfaces that do not display Appropriate Legal Notices, your
|
||||
work need not make them do so.
|
||||
|
||||
A compilation of a covered work with other separate and independent
|
||||
works, which are not by their nature extensions of the covered work,
|
||||
and which are not combined with it such as to form a larger program,
|
||||
in or on a volume of a storage or distribution medium, is called an
|
||||
"aggregate" if the compilation and its resulting copyright are not
|
||||
used to limit the access or legal rights of the compilation's users
|
||||
beyond what the individual works permit. Inclusion of a covered work
|
||||
in an aggregate does not cause this License to apply to the other
|
||||
parts of the aggregate.
|
||||
|
||||
6. Conveying Non-Source Forms.
|
||||
|
||||
You may convey a covered work in object code form under the terms
|
||||
of sections 4 and 5, provided that you also convey the
|
||||
machine-readable Corresponding Source under the terms of this License,
|
||||
in one of these ways:
|
||||
|
||||
a) Convey the object code in, or embodied in, a physical product
|
||||
(including a physical distribution medium), accompanied by the
|
||||
Corresponding Source fixed on a durable physical medium
|
||||
customarily used for software interchange.
|
||||
|
||||
b) Convey the object code in, or embodied in, a physical product
|
||||
(including a physical distribution medium), accompanied by a
|
||||
written offer, valid for at least three years and valid for as
|
||||
long as you offer spare parts or customer support for that product
|
||||
model, to give anyone who possesses the object code either (1) a
|
||||
copy of the Corresponding Source for all the software in the
|
||||
product that is covered by this License, on a durable physical
|
||||
medium customarily used for software interchange, for a price no
|
||||
more than your reasonable cost of physically performing this
|
||||
conveying of source, or (2) access to copy the
|
||||
Corresponding Source from a network server at no charge.
|
||||
|
||||
c) Convey individual copies of the object code with a copy of the
|
||||
written offer to provide the Corresponding Source. This
|
||||
alternative is allowed only occasionally and noncommercially, and
|
||||
only if you received the object code with such an offer, in accord
|
||||
with subsection 6b.
|
||||
|
||||
d) Convey the object code by offering access from a designated
|
||||
place (gratis or for a charge), and offer equivalent access to the
|
||||
Corresponding Source in the same way through the same place at no
|
||||
further charge. You need not require recipients to copy the
|
||||
Corresponding Source along with the object code. If the place to
|
||||
copy the object code is a network server, the Corresponding Source
|
||||
may be on a different server (operated by you or a third party)
|
||||
that supports equivalent copying facilities, provided you maintain
|
||||
clear directions next to the object code saying where to find the
|
||||
Corresponding Source. Regardless of what server hosts the
|
||||
Corresponding Source, you remain obligated to ensure that it is
|
||||
available for as long as needed to satisfy these requirements.
|
||||
|
||||
e) Convey the object code using peer-to-peer transmission, provided
|
||||
you inform other peers where the object code and Corresponding
|
||||
Source of the work are being offered to the general public at no
|
||||
charge under subsection 6d.
|
||||
|
||||
A separable portion of the object code, whose source code is excluded
|
||||
from the Corresponding Source as a System Library, need not be
|
||||
included in conveying the object code work.
|
||||
|
||||
A "User Product" is either (1) a "consumer product", which means any
|
||||
tangible personal property which is normally used for personal, family,
|
||||
or household purposes, or (2) anything designed or sold for incorporation
|
||||
into a dwelling. In determining whether a product is a consumer product,
|
||||
doubtful cases shall be resolved in favor of coverage. For a particular
|
||||
product received by a particular user, "normally used" refers to a
|
||||
typical or common use of that class of product, regardless of the status
|
||||
of the particular user or of the way in which the particular user
|
||||
actually uses, or expects or is expected to use, the product. A product
|
||||
is a consumer product regardless of whether the product has substantial
|
||||
commercial, industrial or non-consumer uses, unless such uses represent
|
||||
the only significant mode of use of the product.
|
||||
|
||||
"Installation Information" for a User Product means any methods,
|
||||
procedures, authorization keys, or other information required to install
|
||||
and execute modified versions of a covered work in that User Product from
|
||||
a modified version of its Corresponding Source. The information must
|
||||
suffice to ensure that the continued functioning of the modified object
|
||||
code is in no case prevented or interfered with solely because
|
||||
modification has been made.
|
||||
|
||||
If you convey an object code work under this section in, or with, or
|
||||
specifically for use in, a User Product, and the conveying occurs as
|
||||
part of a transaction in which the right of possession and use of the
|
||||
User Product is transferred to the recipient in perpetuity or for a
|
||||
fixed term (regardless of how the transaction is characterized), the
|
||||
Corresponding Source conveyed under this section must be accompanied
|
||||
by the Installation Information. But this requirement does not apply
|
||||
if neither you nor any third party retains the ability to install
|
||||
modified object code on the User Product (for example, the work has
|
||||
been installed in ROM).
|
||||
|
||||
The requirement to provide Installation Information does not include a
|
||||
requirement to continue to provide support service, warranty, or updates
|
||||
for a work that has been modified or installed by the recipient, or for
|
||||
the User Product in which it has been modified or installed. Access to a
|
||||
network may be denied when the modification itself materially and
|
||||
adversely affects the operation of the network or violates the rules and
|
||||
protocols for communication across the network.
|
||||
|
||||
Corresponding Source conveyed, and Installation Information provided,
|
||||
in accord with this section must be in a format that is publicly
|
||||
documented (and with an implementation available to the public in
|
||||
source code form), and must require no special password or key for
|
||||
unpacking, reading or copying.
|
||||
|
||||
7. Additional Terms.
|
||||
|
||||
"Additional permissions" are terms that supplement the terms of this
|
||||
License by making exceptions from one or more of its conditions.
|
||||
Additional permissions that are applicable to the entire Program shall
|
||||
be treated as though they were included in this License, to the extent
|
||||
that they are valid under applicable law. If additional permissions
|
||||
apply only to part of the Program, that part may be used separately
|
||||
under those permissions, but the entire Program remains governed by
|
||||
this License without regard to the additional permissions.
|
||||
|
||||
When you convey a copy of a covered work, you may at your option
|
||||
remove any additional permissions from that copy, or from any part of
|
||||
it. (Additional permissions may be written to require their own
|
||||
removal in certain cases when you modify the work.) You may place
|
||||
additional permissions on material, added by you to a covered work,
|
||||
for which you have or can give appropriate copyright permission.
|
||||
|
||||
Notwithstanding any other provision of this License, for material you
|
||||
add to a covered work, you may (if authorized by the copyright holders of
|
||||
that material) supplement the terms of this License with terms:
|
||||
|
||||
a) Disclaiming warranty or limiting liability differently from the
|
||||
terms of sections 15 and 16 of this License; or
|
||||
|
||||
b) Requiring preservation of specified reasonable legal notices or
|
||||
author attributions in that material or in the Appropriate Legal
|
||||
Notices displayed by works containing it; or
|
||||
|
||||
c) Prohibiting misrepresentation of the origin of that material, or
|
||||
requiring that modified versions of such material be marked in
|
||||
reasonable ways as different from the original version; or
|
||||
|
||||
d) Limiting the use for publicity purposes of names of licensors or
|
||||
authors of the material; or
|
||||
|
||||
e) Declining to grant rights under trademark law for use of some
|
||||
trade names, trademarks, or service marks; or
|
||||
|
||||
f) Requiring indemnification of licensors and authors of that
|
||||
material by anyone who conveys the material (or modified versions of
|
||||
it) with contractual assumptions of liability to the recipient, for
|
||||
any liability that these contractual assumptions directly impose on
|
||||
those licensors and authors.
|
||||
|
||||
All other non-permissive additional terms are considered "further
|
||||
restrictions" within the meaning of section 10. If the Program as you
|
||||
received it, or any part of it, contains a notice stating that it is
|
||||
governed by this License along with a term that is a further
|
||||
restriction, you may remove that term. If a license document contains
|
||||
a further restriction but permits relicensing or conveying under this
|
||||
License, you may add to a covered work material governed by the terms
|
||||
of that license document, provided that the further restriction does
|
||||
not survive such relicensing or conveying.
|
||||
|
||||
If you add terms to a covered work in accord with this section, you
|
||||
must place, in the relevant source files, a statement of the
|
||||
additional terms that apply to those files, or a notice indicating
|
||||
where to find the applicable terms.
|
||||
|
||||
Additional terms, permissive or non-permissive, may be stated in the
|
||||
form of a separately written license, or stated as exceptions;
|
||||
the above requirements apply either way.
|
||||
|
||||
8. Termination.
|
||||
|
||||
You may not propagate or modify a covered work except as expressly
|
||||
provided under this License. Any attempt otherwise to propagate or
|
||||
modify it is void, and will automatically terminate your rights under
|
||||
this License (including any patent licenses granted under the third
|
||||
paragraph of section 11).
|
||||
|
||||
However, if you cease all violation of this License, then your
|
||||
license from a particular copyright holder is reinstated (a)
|
||||
provisionally, unless and until the copyright holder explicitly and
|
||||
finally terminates your license, and (b) permanently, if the copyright
|
||||
holder fails to notify you of the violation by some reasonable means
|
||||
prior to 60 days after the cessation.
|
||||
|
||||
Moreover, your license from a particular copyright holder is
|
||||
reinstated permanently if the copyright holder notifies you of the
|
||||
violation by some reasonable means, this is the first time you have
|
||||
received notice of violation of this License (for any work) from that
|
||||
copyright holder, and you cure the violation prior to 30 days after
|
||||
your receipt of the notice.
|
||||
|
||||
Termination of your rights under this section does not terminate the
|
||||
licenses of parties who have received copies or rights from you under
|
||||
this License. If your rights have been terminated and not permanently
|
||||
reinstated, you do not qualify to receive new licenses for the same
|
||||
material under section 10.
|
||||
|
||||
9. Acceptance Not Required for Having Copies.
|
||||
|
||||
You are not required to accept this License in order to receive or
|
||||
run a copy of the Program. Ancillary propagation of a covered work
|
||||
occurring solely as a consequence of using peer-to-peer transmission
|
||||
to receive a copy likewise does not require acceptance. However,
|
||||
nothing other than this License grants you permission to propagate or
|
||||
modify any covered work. These actions infringe copyright if you do
|
||||
not accept this License. Therefore, by modifying or propagating a
|
||||
covered work, you indicate your acceptance of this License to do so.
|
||||
|
||||
10. Automatic Licensing of Downstream Recipients.
|
||||
|
||||
Each time you convey a covered work, the recipient automatically
|
||||
receives a license from the original licensors, to run, modify and
|
||||
propagate that work, subject to this License. You are not responsible
|
||||
for enforcing compliance by third parties with this License.
|
||||
|
||||
An "entity transaction" is a transaction transferring control of an
|
||||
organization, or substantially all assets of one, or subdividing an
|
||||
organization, or merging organizations. If propagation of a covered
|
||||
work results from an entity transaction, each party to that
|
||||
transaction who receives a copy of the work also receives whatever
|
||||
licenses to the work the party's predecessor in interest had or could
|
||||
give under the previous paragraph, plus a right to possession of the
|
||||
Corresponding Source of the work from the predecessor in interest, if
|
||||
the predecessor has it or can get it with reasonable efforts.
|
||||
|
||||
You may not impose any further restrictions on the exercise of the
|
||||
rights granted or affirmed under this License. For example, you may
|
||||
not impose a license fee, royalty, or other charge for exercise of
|
||||
rights granted under this License, and you may not initiate litigation
|
||||
(including a cross-claim or counterclaim in a lawsuit) alleging that
|
||||
any patent claim is infringed by making, using, selling, offering for
|
||||
sale, or importing the Program or any portion of it.
|
||||
|
||||
11. Patents.
|
||||
|
||||
A "contributor" is a copyright holder who authorizes use under this
|
||||
License of the Program or a work on which the Program is based. The
|
||||
work thus licensed is called the contributor's "contributor version".
|
||||
|
||||
A contributor's "essential patent claims" are all patent claims
|
||||
owned or controlled by the contributor, whether already acquired or
|
||||
hereafter acquired, that would be infringed by some manner, permitted
|
||||
by this License, of making, using, or selling its contributor version,
|
||||
but do not include claims that would be infringed only as a
|
||||
consequence of further modification of the contributor version. For
|
||||
purposes of this definition, "control" includes the right to grant
|
||||
patent sublicenses in a manner consistent with the requirements of
|
||||
this License.
|
||||
|
||||
Each contributor grants you a non-exclusive, worldwide, royalty-free
|
||||
patent license under the contributor's essential patent claims, to
|
||||
make, use, sell, offer for sale, import and otherwise run, modify and
|
||||
propagate the contents of its contributor version.
|
||||
|
||||
In the following three paragraphs, a "patent license" is any express
|
||||
agreement or commitment, however denominated, not to enforce a patent
|
||||
(such as an express permission to practice a patent or covenant not to
|
||||
sue for patent infringement). To "grant" such a patent license to a
|
||||
party means to make such an agreement or commitment not to enforce a
|
||||
patent against the party.
|
||||
|
||||
If you convey a covered work, knowingly relying on a patent license,
|
||||
and the Corresponding Source of the work is not available for anyone
|
||||
to copy, free of charge and under the terms of this License, through a
|
||||
publicly available network server or other readily accessible means,
|
||||
then you must either (1) cause the Corresponding Source to be so
|
||||
available, or (2) arrange to deprive yourself of the benefit of the
|
||||
patent license for this particular work, or (3) arrange, in a manner
|
||||
consistent with the requirements of this License, to extend the patent
|
||||
license to downstream recipients. "Knowingly relying" means you have
|
||||
actual knowledge that, but for the patent license, your conveying the
|
||||
covered work in a country, or your recipient's use of the covered work
|
||||
in a country, would infringe one or more identifiable patents in that
|
||||
country that you have reason to believe are valid.
|
||||
|
||||
If, pursuant to or in connection with a single transaction or
|
||||
arrangement, you convey, or propagate by procuring conveyance of, a
|
||||
covered work, and grant a patent license to some of the parties
|
||||
receiving the covered work authorizing them to use, propagate, modify
|
||||
or convey a specific copy of the covered work, then the patent license
|
||||
you grant is automatically extended to all recipients of the covered
|
||||
work and works based on it.
|
||||
|
||||
A patent license is "discriminatory" if it does not include within
|
||||
the scope of its coverage, prohibits the exercise of, or is
|
||||
conditioned on the non-exercise of one or more of the rights that are
|
||||
specifically granted under this License. You may not convey a covered
|
||||
work if you are a party to an arrangement with a third party that is
|
||||
in the business of distributing software, under which you make payment
|
||||
to the third party based on the extent of your activity of conveying
|
||||
the work, and under which the third party grants, to any of the
|
||||
parties who would receive the covered work from you, a discriminatory
|
||||
patent license (a) in connection with copies of the covered work
|
||||
conveyed by you (or copies made from those copies), or (b) primarily
|
||||
for and in connection with specific products or compilations that
|
||||
contain the covered work, unless you entered into that arrangement,
|
||||
or that patent license was granted, prior to 28 March 2007.
|
||||
|
||||
Nothing in this License shall be construed as excluding or limiting
|
||||
any implied license or other defenses to infringement that may
|
||||
otherwise be available to you under applicable patent law.
|
||||
|
||||
12. No Surrender of Others' Freedom.
|
||||
|
||||
If conditions are imposed on you (whether by court order, agreement or
|
||||
otherwise) that contradict the conditions of this License, they do not
|
||||
excuse you from the conditions of this License. If you cannot convey a
|
||||
covered work so as to satisfy simultaneously your obligations under this
|
||||
License and any other pertinent obligations, then as a consequence you may
|
||||
not convey it at all. For example, if you agree to terms that obligate you
|
||||
to collect a royalty for further conveying from those to whom you convey
|
||||
the Program, the only way you could satisfy both those terms and this
|
||||
License would be to refrain entirely from conveying the Program.
|
||||
|
||||
13. Remote Network Interaction; Use with the GNU General Public License.
|
||||
|
||||
Notwithstanding any other provision of this License, if you modify the
|
||||
Program, your modified version must prominently offer all users
|
||||
interacting with it remotely through a computer network (if your version
|
||||
supports such interaction) an opportunity to receive the Corresponding
|
||||
Source of your version by providing access to the Corresponding Source
|
||||
from a network server at no charge, through some standard or customary
|
||||
means of facilitating copying of software. This Corresponding Source
|
||||
shall include the Corresponding Source for any work covered by version 3
|
||||
of the GNU General Public License that is incorporated pursuant to the
|
||||
following paragraph.
|
||||
|
||||
Notwithstanding any other provision of this License, you have
|
||||
permission to link or combine any covered work with a work licensed
|
||||
under version 3 of the GNU General Public License into a single
|
||||
combined work, and to convey the resulting work. The terms of this
|
||||
License will continue to apply to the part which is the covered work,
|
||||
but the work with which it is combined will remain governed by version
|
||||
3 of the GNU General Public License.
|
||||
|
||||
14. Revised Versions of this License.
|
||||
|
||||
The Free Software Foundation may publish revised and/or new versions of
|
||||
the GNU Affero General Public License from time to time. Such new versions
|
||||
will be similar in spirit to the present version, but may differ in detail to
|
||||
address new problems or concerns.
|
||||
|
||||
Each version is given a distinguishing version number. If the
|
||||
Program specifies that a certain numbered version of the GNU Affero General
|
||||
Public License "or any later version" applies to it, you have the
|
||||
option of following the terms and conditions either of that numbered
|
||||
version or of any later version published by the Free Software
|
||||
Foundation. If the Program does not specify a version number of the
|
||||
GNU Affero General Public License, you may choose any version ever published
|
||||
by the Free Software Foundation.
|
||||
|
||||
If the Program specifies that a proxy can decide which future
|
||||
versions of the GNU Affero General Public License can be used, that proxy's
|
||||
public statement of acceptance of a version permanently authorizes you
|
||||
to choose that version for the Program.
|
||||
|
||||
Later license versions may give you additional or different
|
||||
permissions. However, no additional obligations are imposed on any
|
||||
author or copyright holder as a result of your choosing to follow a
|
||||
later version.
|
||||
|
||||
15. Disclaimer of Warranty.
|
||||
|
||||
THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
|
||||
APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
|
||||
HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
|
||||
OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
|
||||
THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
||||
PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
|
||||
IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
|
||||
ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
|
||||
|
||||
16. Limitation of Liability.
|
||||
|
||||
IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
|
||||
WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
|
||||
THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
|
||||
GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
|
||||
USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
|
||||
DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
|
||||
PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
|
||||
EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
|
||||
SUCH DAMAGES.
|
||||
|
||||
17. Interpretation of Sections 15 and 16.
|
||||
|
||||
If the disclaimer of warranty and limitation of liability provided
|
||||
above cannot be given local legal effect according to their terms,
|
||||
reviewing courts shall apply local law that most closely approximates
|
||||
an absolute waiver of all civil liability in connection with the
|
||||
Program, unless a warranty or assumption of liability accompanies a
|
||||
copy of the Program in return for a fee.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
How to Apply These Terms to Your New Programs
|
||||
|
||||
If you develop a new program, and you want it to be of the greatest
|
||||
possible use to the public, the best way to achieve this is to make it
|
||||
free software which everyone can redistribute and change under these terms.
|
||||
|
||||
To do so, attach the following notices to the program. It is safest
|
||||
to attach them to the start of each source file to most effectively
|
||||
state the exclusion of warranty; and each file should have at least
|
||||
the "copyright" line and a pointer to where the full notice is found.
|
||||
|
||||
<one line to give the program's name and a brief idea of what it does.>
|
||||
Copyright (C) <year> <name of author>
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as published
|
||||
by the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
Also add information on how to contact you by electronic and paper mail.
|
||||
|
||||
If your software can interact with users remotely through a computer
|
||||
network, you should also make sure that it provides a way for users to
|
||||
get its source. For example, if your program is a web application, its
|
||||
interface could display a "Source" link that leads users to an archive
|
||||
of the code. There are many ways you could offer source, and different
|
||||
solutions will be better for different programs; see section 13 for the
|
||||
specific requirements.
|
||||
|
||||
You should also get your employer (if you work as a programmer) or school,
|
||||
if any, to sign a "copyright disclaimer" for the program, if necessary.
|
||||
For more information on this, and how to apply and follow the GNU AGPL, see
|
||||
<https://www.gnu.org/licenses/>.
|
||||
|
|
@ -0,0 +1,82 @@
|
|||
```regex
|
||||
██████╗ ██████╗ ██████╗ ████████╗██╗ █████╗
|
||||
██╔══██╗██╔═══██╗██╔══██╗╚══██╔══╝██║██╔══██╗
|
||||
██████╔╝██║ ██║██████╔╝ ██║ ██║███████║
|
||||
██╔═══╝ ██║ ██║██╔══██╗ ██║ ██║██╔══██║
|
||||
██║ ╚██████╔╝██║ ██║ ██║ ██║██║ ██║
|
||||
╚═╝ ╚═════╝ ╚═╝ ╚═╝ ╚═╝ ╚═╝╚═╝ ╚═╝
|
||||
```
|
||||
|
||||
[](https://github.com/CarterPerez-dev/Cybersecurity-Projects/tree/main/PROJECTS/intermediate/secrets-scanner)
|
||||
[](https://go.dev)
|
||||
[](https://www.gnu.org/licenses/agpl-3.0)
|
||||
[](https://haveibeenpwned.com/API/v3)
|
||||
|
||||
> Secrets scanner for codebases and git repositories, written in Go.
|
||||
|
||||
*This is a quick overview. Security theory, architecture, and full walkthroughs are in the [learn modules](#learn).*
|
||||
|
||||
## What It Does
|
||||
|
||||
- 150 detection rules covering AWS, GitHub, GitLab, GCP, Azure, Slack, Stripe, Twilio, SendGrid, SSH/PGP keys, passwords, connection strings, JWTs, and 100+ more
|
||||
- Shannon entropy analysis for detecting high-randomness strings
|
||||
- HIBP breach verification via k-anonymity protocol (your secrets never leave your machine)
|
||||
- Directory scanning and full git history scanning (branches, depth, date ranges)
|
||||
- Output as colored terminal tables, JSON, or SARIF v2.1.0
|
||||
- 5-layer false positive defense: keyword pre-filter, structural validation, stopwords, allowlists, entropy
|
||||
- Concurrent pipeline with bounded worker pools
|
||||
- TOML configuration via `.portia.toml` or `pyproject.toml`
|
||||
|
||||
## Install
|
||||
|
||||
```bash
|
||||
curl -fsSL https://raw.githubusercontent.com/CarterPerez-dev/portia/main/install.sh | bash
|
||||
```
|
||||
|
||||
Or with Go:
|
||||
|
||||
```bash
|
||||
go install github.com/CarterPerez-dev/portia/cmd/portia@latest
|
||||
```
|
||||
|
||||
## Quick Start
|
||||
|
||||
```bash
|
||||
portia scan .
|
||||
```
|
||||
|
||||
> [!TIP]
|
||||
> This project uses [`just`](https://github.com/casey/just) as a command runner. Type `just` to see all available commands.
|
||||
>
|
||||
> Install: `curl -sSf https://just.systems/install.sh | bash -s -- --to ~/.local/bin`
|
||||
|
||||
## Commands
|
||||
|
||||
| Command | Description |
|
||||
|---------|-------------|
|
||||
| `portia scan [path]` | Scan a directory for secrets |
|
||||
| `portia git [repo]` | Scan git history for secrets |
|
||||
| `portia init` | Initialize `.portia.toml` configuration |
|
||||
| `portia pyproject` | Create `pyproject.toml` with `[tool.portia]` config |
|
||||
| `portia config rules` | List all 150 detection rules |
|
||||
| `portia config show` | Show active configuration |
|
||||
|
||||
**Flags:** `--format` (terminal/json/sarif), `--verbose`, `--no-color`, `--exclude`, `--max-size`, `--hibp`, `--config`
|
||||
|
||||
**Git flags:** `--branch`, `--since`, `--depth`, `--staged`
|
||||
|
||||
## Learn
|
||||
|
||||
This project includes step-by-step learning materials covering security theory, architecture, and implementation.
|
||||
|
||||
| Module | Topic |
|
||||
|--------|-------|
|
||||
| [00 - Overview](learn/00-OVERVIEW.md) | Prerequisites and quick start |
|
||||
| [01 - Concepts](learn/01-CONCEPTS.md) | Secret sprawl, entropy, and breach databases |
|
||||
| [02 - Architecture](learn/02-ARCHITECTURE.md) | System design and data flow |
|
||||
| [03 - Implementation](learn/03-IMPLEMENTATION.md) | Code walkthrough |
|
||||
| [04 - Challenges](learn/04-CHALLENGES.md) | Extension ideas and exercises |
|
||||
|
||||
## License
|
||||
|
||||
AGPL 3.0
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
// ©AngelaMos | 2026
|
||||
// main.go
|
||||
|
||||
package main
|
||||
|
||||
import "github.com/CarterPerez-dev/portia/internal/cli"
|
||||
|
||||
func main() {
|
||||
cli.Execute()
|
||||
}
|
||||
|
|
@ -0,0 +1,45 @@
|
|||
module github.com/CarterPerez-dev/portia
|
||||
|
||||
go 1.24.4
|
||||
|
||||
require (
|
||||
github.com/fatih/color v1.18.0
|
||||
github.com/go-git/go-git/v5 v5.16.5
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7
|
||||
github.com/pelletier/go-toml/v2 v2.2.4
|
||||
github.com/sony/gobreaker/v2 v2.4.0
|
||||
github.com/spf13/cobra v1.10.2
|
||||
github.com/stretchr/testify v1.11.1
|
||||
golang.org/x/sync v0.19.0
|
||||
golang.org/x/time v0.14.0
|
||||
)
|
||||
|
||||
require (
|
||||
dario.cat/mergo v1.0.2 // indirect
|
||||
github.com/Microsoft/go-winio v0.6.2 // indirect
|
||||
github.com/ProtonMail/go-crypto v1.3.0 // indirect
|
||||
github.com/cloudflare/circl v1.6.3 // indirect
|
||||
github.com/cyphar/filepath-securejoin v0.6.1 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/emirpasic/gods v1.18.1 // indirect
|
||||
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
|
||||
github.com/go-git/go-billy/v5 v5.7.0 // indirect
|
||||
github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
|
||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
|
||||
github.com/kevinburke/ssh_config v1.6.0 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.3.0 // indirect
|
||||
github.com/mattn/go-colorable v0.1.14 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/pjbgf/sha1cd v0.5.0 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/sergi/go-diff v1.4.0 // indirect
|
||||
github.com/skeema/knownhosts v1.3.2 // indirect
|
||||
github.com/spf13/pflag v1.0.10 // indirect
|
||||
github.com/xanzy/ssh-agent v0.3.3 // indirect
|
||||
golang.org/x/crypto v0.48.0 // indirect
|
||||
golang.org/x/net v0.50.0 // indirect
|
||||
golang.org/x/sys v0.41.0 // indirect
|
||||
gopkg.in/warnings.v0 v0.1.2 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
)
|
||||
|
|
@ -0,0 +1,131 @@
|
|||
dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8=
|
||||
dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA=
|
||||
github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=
|
||||
github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
|
||||
github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
|
||||
github.com/ProtonMail/go-crypto v1.3.0 h1:ILq8+Sf5If5DCpHQp4PbZdS1J7HDFRXz/+xKBiRGFrw=
|
||||
github.com/ProtonMail/go-crypto v1.3.0/go.mod h1:9whxjD8Rbs29b4XWbB8irEcE8KHMqaR2e7GWU1R+/PE=
|
||||
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8=
|
||||
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4=
|
||||
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
|
||||
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
|
||||
github.com/cloudflare/circl v1.6.3 h1:9GPOhQGF9MCYUeXyMYlqTR6a5gTrgR/fBLXvUgtVcg8=
|
||||
github.com/cloudflare/circl v1.6.3/go.mod h1:2eXP6Qfat4O/Yhh8BznvKnJ+uzEoTQ6jVKJRn81BiS4=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
|
||||
github.com/cyphar/filepath-securejoin v0.6.1 h1:5CeZ1jPXEiYt3+Z6zqprSAgSWiggmpVyciv8syjIpVE=
|
||||
github.com/cyphar/filepath-securejoin v0.6.1/go.mod h1:A8hd4EnAeyujCJRrICiOWqjS1AX0a9kM5XL+NwKoYSc=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o=
|
||||
github.com/elazarl/goproxy v1.7.2/go.mod h1:82vkLNir0ALaW14Rc399OTTjyNREgmdL2cVoIbS6XaE=
|
||||
github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
|
||||
github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ=
|
||||
github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM=
|
||||
github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU=
|
||||
github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c=
|
||||
github.com/gliderlabs/ssh v0.3.8/go.mod h1:xYoytBv1sV0aL3CavoDuJIQNURXkkfPA/wxQ1pL1fAU=
|
||||
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI=
|
||||
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic=
|
||||
github.com/go-git/go-billy/v5 v5.7.0 h1:83lBUJhGWhYp0ngzCMSgllhUSuoHP1iEWYjsPl9nwqM=
|
||||
github.com/go-git/go-billy/v5 v5.7.0/go.mod h1:/1IUejTKH8xipsAcdfcSAlUlo2J7lkYV8GTKxAT/L3E=
|
||||
github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4=
|
||||
github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII=
|
||||
github.com/go-git/go-git/v5 v5.16.5 h1:mdkuqblwr57kVfXri5TTH+nMFLNUxIj9Z7F5ykFbw5s=
|
||||
github.com/go-git/go-git/v5 v5.16.5/go.mod h1:QOMLpNf1qxuSY4StA/ArOdfFR2TrKEjJiye2kel2m+M=
|
||||
github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ=
|
||||
github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
|
||||
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
||||
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
|
||||
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
|
||||
github.com/kevinburke/ssh_config v1.6.0 h1:J1FBfmuVosPHf5GRdltRLhPJtJpTlMdKTBjRgTaQBFY=
|
||||
github.com/kevinburke/ssh_config v1.6.0/go.mod h1:q2RIzfka+BXARoNexmF9gkxEX7DmvbW9P4hIVx2Kg4M=
|
||||
github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
|
||||
github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE=
|
||||
github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
|
||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k=
|
||||
github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY=
|
||||
github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
|
||||
github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
|
||||
github.com/pjbgf/sha1cd v0.5.0 h1:a+UkboSi1znleCDUNT3M5YxjOnN1fz2FhN48FlwCxs0=
|
||||
github.com/pjbgf/sha1cd v0.5.0/go.mod h1:lhpGlyHLpQZoxMv8HcgXvZEhcGs0PG/vsZnEJ7H0iCM=
|
||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
|
||||
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
|
||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw=
|
||||
github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4=
|
||||
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
|
||||
github.com/skeema/knownhosts v1.3.2 h1:EDL9mgf4NzwMXCTfaxSD/o/a5fxDw/xL9nkU28JjdBg=
|
||||
github.com/skeema/knownhosts v1.3.2/go.mod h1:bEg3iQAuw+jyiw+484wwFJoKSLwcfd7fqRy+N0QTiow=
|
||||
github.com/sony/gobreaker/v2 v2.4.0 h1:g2KJRW1Ubty3+ZOcSEUN7K+REQJdN6yo6XvaML+jptg=
|
||||
github.com/sony/gobreaker/v2 v2.4.0/go.mod h1:pTyFJgcZ3h2tdQVLZZruK2C0eoFL1fb/G83wK1ZQl+s=
|
||||
github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU=
|
||||
github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4=
|
||||
github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
|
||||
github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM=
|
||||
github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
|
||||
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
||||
golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts=
|
||||
golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos=
|
||||
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8=
|
||||
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY=
|
||||
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60=
|
||||
golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM=
|
||||
golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
|
||||
golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
|
||||
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=
|
||||
golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg=
|
||||
golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM=
|
||||
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=
|
||||
golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
|
||||
golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
|
||||
golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME=
|
||||
gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
|
|
@ -0,0 +1,216 @@
|
|||
#!/usr/bin/env bash
|
||||
# ©AngelaMos | 2026
|
||||
# install.sh
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
REPO_OWNER="CarterPerez-dev"
|
||||
REPO_NAME="portia"
|
||||
BINARY="portia"
|
||||
INSTALL_DIR="${PORTIA_INSTALL_DIR:-$HOME/.portia/bin}"
|
||||
VERSION="${PORTIA_VERSION:-}"
|
||||
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
CYAN='\033[0;36m'
|
||||
BOLD='\033[1m'
|
||||
DIM='\033[2m'
|
||||
NC='\033[0m'
|
||||
|
||||
info() { echo -e " ${GREEN}+${NC} $1"; }
|
||||
warn() { echo -e " ${YELLOW}!${NC} $1"; }
|
||||
fail() { echo -e " ${RED}x${NC} $1"; exit 1; }
|
||||
header() { echo -e "\n${BOLD}${CYAN}--- $1 ---${NC}\n"; }
|
||||
|
||||
TMP_DIR=""
|
||||
cleanup() { [[ -n "$TMP_DIR" ]] && rm -rf "$TMP_DIR"; }
|
||||
trap cleanup EXIT
|
||||
|
||||
# =========================================================================
|
||||
# Banner
|
||||
# =========================================================================
|
||||
|
||||
echo -e "${BOLD}"
|
||||
echo -e " ${RED} ▀█▀ ▄▀▄ █▀▄ ▀█▀ █ ▄▀▄ ${NC}"
|
||||
echo -e " ${CYAN} █ ▀▄▀ █▀▄ █ █ █▀█ ${NC}"
|
||||
echo -e "${NC}"
|
||||
echo -e " ${DIM}Secrets scanner for codebases and git repositories${NC}"
|
||||
|
||||
# =========================================================================
|
||||
# Detect System
|
||||
# =========================================================================
|
||||
|
||||
header "Detecting system"
|
||||
|
||||
OS="$(uname -s)"
|
||||
ARCH="$(uname -m)"
|
||||
|
||||
case "$OS" in
|
||||
Linux) OS="linux" ;;
|
||||
Darwin) OS="darwin" ;;
|
||||
MINGW*|MSYS*|CYGWIN*) fail "Windows is not supported. Use: go install github.com/${REPO_OWNER}/${REPO_NAME}/cmd/portia@latest" ;;
|
||||
*) fail "Unsupported OS: $OS" ;;
|
||||
esac
|
||||
|
||||
case "$ARCH" in
|
||||
x86_64|amd64) ARCH="amd64" ;;
|
||||
aarch64|arm64) ARCH="arm64" ;;
|
||||
*) fail "Unsupported architecture: $ARCH" ;;
|
||||
esac
|
||||
|
||||
info "System: ${OS}/${ARCH}"
|
||||
|
||||
# =========================================================================
|
||||
# Resolve Version
|
||||
# =========================================================================
|
||||
|
||||
if [[ -z "$VERSION" ]]; then
|
||||
header "Fetching latest release"
|
||||
|
||||
if command -v curl &>/dev/null; then
|
||||
VERSION=$(curl -fsSL "https://api.github.com/repos/${REPO_OWNER}/${REPO_NAME}/releases/latest" 2>/dev/null \
|
||||
| grep '"tag_name":' \
|
||||
| sed -E 's/.*"([^"]+)".*/\1/') || true
|
||||
elif command -v wget &>/dev/null; then
|
||||
VERSION=$(wget -qO- "https://api.github.com/repos/${REPO_OWNER}/${REPO_NAME}/releases/latest" 2>/dev/null \
|
||||
| grep '"tag_name":' \
|
||||
| sed -E 's/.*"([^"]+)".*/\1/') || true
|
||||
fi
|
||||
fi
|
||||
|
||||
# =========================================================================
|
||||
# Install: Pre-built Binary
|
||||
# =========================================================================
|
||||
|
||||
INSTALLED=false
|
||||
|
||||
if [[ -n "$VERSION" ]]; then
|
||||
info "Version: ${VERSION}"
|
||||
header "Downloading pre-built binary"
|
||||
|
||||
ARCHIVE="${BINARY}_${VERSION#v}_${OS}_${ARCH}.tar.gz"
|
||||
URL="https://github.com/${REPO_OWNER}/${REPO_NAME}/releases/download/${VERSION}/${ARCHIVE}"
|
||||
|
||||
TMP_DIR=$(mktemp -d)
|
||||
|
||||
DOWNLOAD_OK=false
|
||||
if command -v curl &>/dev/null; then
|
||||
curl -fsSL "$URL" -o "$TMP_DIR/archive.tar.gz" 2>/dev/null && DOWNLOAD_OK=true
|
||||
elif command -v wget &>/dev/null; then
|
||||
wget -q "$URL" -O "$TMP_DIR/archive.tar.gz" 2>/dev/null && DOWNLOAD_OK=true
|
||||
else
|
||||
fail "Neither curl nor wget found"
|
||||
fi
|
||||
|
||||
if [[ "$DOWNLOAD_OK" == "true" ]]; then
|
||||
tar -xzf "$TMP_DIR/archive.tar.gz" -C "$TMP_DIR"
|
||||
mkdir -p "$INSTALL_DIR"
|
||||
mv "$TMP_DIR/$BINARY" "$INSTALL_DIR/"
|
||||
chmod +x "$INSTALL_DIR/$BINARY"
|
||||
INSTALLED=true
|
||||
info "Installed to ${INSTALL_DIR}/${BINARY}"
|
||||
else
|
||||
warn "Pre-built binary not available for ${OS}/${ARCH}"
|
||||
fi
|
||||
fi
|
||||
|
||||
# =========================================================================
|
||||
# Fallback: go install
|
||||
# =========================================================================
|
||||
|
||||
if [[ "$INSTALLED" == "false" ]]; then
|
||||
if command -v go &>/dev/null; then
|
||||
GO_VER=$(go version | awk '{print $3}')
|
||||
header "Building from source (${GO_VER})"
|
||||
info "Running go install..."
|
||||
GOBIN="$INSTALL_DIR" go install "github.com/${REPO_OWNER}/${REPO_NAME}/cmd/portia@latest"
|
||||
INSTALLED=true
|
||||
info "Installed to ${INSTALL_DIR}/${BINARY}"
|
||||
else
|
||||
echo ""
|
||||
fail "No pre-built binary and Go is not installed.
|
||||
|
||||
Option 1 — Install Go, then:
|
||||
go install github.com/${REPO_OWNER}/${REPO_NAME}/cmd/portia@latest
|
||||
|
||||
Option 2 — Install Go:
|
||||
https://go.dev/dl/"
|
||||
fi
|
||||
fi
|
||||
|
||||
# =========================================================================
|
||||
# PATH Setup
|
||||
# =========================================================================
|
||||
|
||||
header "Configuring PATH"
|
||||
|
||||
PATH_UPDATED=false
|
||||
|
||||
case ":$PATH:" in
|
||||
*":${INSTALL_DIR}:"*)
|
||||
info "${INSTALL_DIR} already in PATH"
|
||||
PATH_UPDATED=true
|
||||
;;
|
||||
esac
|
||||
|
||||
if [[ "$PATH_UPDATED" == "false" ]]; then
|
||||
CURRENT_SHELL="$(basename "${SHELL:-/bin/bash}")"
|
||||
TARGET=""
|
||||
|
||||
case "$CURRENT_SHELL" in
|
||||
zsh)
|
||||
[[ -f "$HOME/.zshrc" ]] && TARGET="$HOME/.zshrc"
|
||||
;;
|
||||
bash)
|
||||
if [[ -f "$HOME/.bashrc" ]]; then
|
||||
TARGET="$HOME/.bashrc"
|
||||
elif [[ -f "$HOME/.bash_profile" ]]; then
|
||||
TARGET="$HOME/.bash_profile"
|
||||
fi
|
||||
;;
|
||||
fish)
|
||||
mkdir -p "$HOME/.config/fish/conf.d"
|
||||
echo "set -gx PATH \"$INSTALL_DIR\" \$PATH" > "$HOME/.config/fish/conf.d/portia.fish"
|
||||
info "Added to ~/.config/fish/conf.d/portia.fish"
|
||||
PATH_UPDATED=true
|
||||
;;
|
||||
esac
|
||||
|
||||
if [[ "$PATH_UPDATED" == "false" && -z "${TARGET:-}" ]]; then
|
||||
[[ -f "$HOME/.profile" ]] && TARGET="$HOME/.profile"
|
||||
fi
|
||||
|
||||
if [[ "$PATH_UPDATED" == "false" && -n "${TARGET:-}" ]]; then
|
||||
if ! grep -q "$INSTALL_DIR" "$TARGET" 2>/dev/null; then
|
||||
printf '\nexport PATH="%s:$PATH"\n' "$INSTALL_DIR" >> "$TARGET"
|
||||
info "Added to ${TARGET}"
|
||||
else
|
||||
info "Already configured in ${TARGET}"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# =========================================================================
|
||||
# Done
|
||||
# =========================================================================
|
||||
|
||||
echo ""
|
||||
echo -e " ${GREEN}${BOLD}portia installed successfully${NC}"
|
||||
echo ""
|
||||
|
||||
if ! command -v portia &>/dev/null; then
|
||||
warn "Restart your shell or run:"
|
||||
echo -e " ${BOLD}export PATH=\"${INSTALL_DIR}:\$PATH\"${NC}"
|
||||
echo ""
|
||||
fi
|
||||
|
||||
echo -e " ${DIM}Quick start:${NC}"
|
||||
echo ""
|
||||
echo -e " ${CYAN}portia scan [path]${NC} Scan a directory for secrets"
|
||||
echo -e " ${CYAN}portia git [path]${NC} Scan git history for secrets"
|
||||
echo -e " ${CYAN}portia init${NC} Create .portia.toml config"
|
||||
echo -e " ${CYAN}portia config rules${NC} List all 150 detection rules"
|
||||
echo ""
|
||||
echo -e " ${DIM}Docs: https://github.com/${REPO_OWNER}/${REPO_NAME}${NC}"
|
||||
echo ""
|
||||
|
|
@ -0,0 +1,108 @@
|
|||
// ©AngelaMos | 2026
|
||||
// config.go
|
||||
|
||||
package cli
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"github.com/CarterPerez-dev/portia/internal/rules"
|
||||
"github.com/CarterPerez-dev/portia/internal/ui"
|
||||
)
|
||||
|
||||
var configCmd = &cobra.Command{
|
||||
Use: "config",
|
||||
Short: "Show configuration and rule information",
|
||||
}
|
||||
|
||||
var configRulesCmd = &cobra.Command{
|
||||
Use: "rules",
|
||||
Short: "List all available detection rules",
|
||||
RunE: runConfigRules,
|
||||
}
|
||||
|
||||
var configShowCmd = &cobra.Command{
|
||||
Use: "show",
|
||||
Short: "Show active configuration",
|
||||
RunE: runConfigShow,
|
||||
}
|
||||
|
||||
func init() {
|
||||
configCmd.AddCommand(configRulesCmd)
|
||||
configCmd.AddCommand(configShowCmd)
|
||||
}
|
||||
|
||||
func runConfigRules(_ *cobra.Command, _ []string) error {
|
||||
reg := rules.NewRegistry()
|
||||
rules.RegisterBuiltins(reg)
|
||||
|
||||
all := reg.All()
|
||||
fmt.Fprintf(os.Stdout, "\n%s %s\n\n", //nolint:errcheck
|
||||
ui.Shield,
|
||||
ui.CyanBold(fmt.Sprintf("%d detection rules", len(all))),
|
||||
)
|
||||
|
||||
for _, r := range all {
|
||||
sevColor := ui.Cyan
|
||||
switch r.Severity.String() {
|
||||
case "CRITICAL":
|
||||
sevColor = ui.RedBold
|
||||
case "HIGH":
|
||||
sevColor = ui.Red
|
||||
case "MEDIUM":
|
||||
sevColor = ui.Yellow
|
||||
}
|
||||
|
||||
fmt.Fprintf(os.Stdout, " %s %-40s %s %s\n", //nolint:errcheck
|
||||
ui.Diamond,
|
||||
ui.White(r.ID),
|
||||
sevColor(fmt.Sprintf("%-8s", r.Severity)),
|
||||
r.Description,
|
||||
)
|
||||
}
|
||||
|
||||
fmt.Fprintln(os.Stdout) //nolint:errcheck
|
||||
return nil
|
||||
}
|
||||
|
||||
func runConfigShow(_ *cobra.Command, _ []string) error {
|
||||
fmt.Fprintf(os.Stdout, "\n%s Active Configuration\n\n", //nolint:errcheck
|
||||
ui.Arrow)
|
||||
fmt.Fprintf( //nolint:errcheck
|
||||
os.Stdout,
|
||||
" Format: %s\n",
|
||||
format,
|
||||
)
|
||||
fmt.Fprintf( //nolint:errcheck
|
||||
os.Stdout,
|
||||
" Verbose: %t\n",
|
||||
verbose,
|
||||
)
|
||||
fmt.Fprintf( //nolint:errcheck
|
||||
os.Stdout,
|
||||
" No Color: %t\n",
|
||||
noColor,
|
||||
)
|
||||
fmt.Fprintf( //nolint:errcheck
|
||||
os.Stdout,
|
||||
" HIBP: %t\n",
|
||||
enableHIBP,
|
||||
)
|
||||
fmt.Fprintf( //nolint:errcheck
|
||||
os.Stdout,
|
||||
" Max Size: %d bytes\n",
|
||||
maxSize,
|
||||
)
|
||||
if len(excludes) > 0 {
|
||||
fmt.Fprintf(os.Stdout, " Excludes: %v\n", excludes) //nolint:errcheck
|
||||
}
|
||||
if cfg != nil && len(cfg.Rules.Disable) > 0 {
|
||||
fmt.Fprintf(os.Stdout, " Disabled: %v\n", //nolint:errcheck
|
||||
cfg.Rules.Disable)
|
||||
}
|
||||
fmt.Fprintln(os.Stdout) //nolint:errcheck
|
||||
return nil
|
||||
}
|
||||
|
|
@ -0,0 +1,72 @@
|
|||
// ©AngelaMos | 2026
|
||||
// git.go
|
||||
|
||||
package cli
|
||||
|
||||
import (
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"github.com/CarterPerez-dev/portia/internal/rules"
|
||||
"github.com/CarterPerez-dev/portia/internal/source"
|
||||
"github.com/CarterPerez-dev/portia/internal/ui"
|
||||
)
|
||||
|
||||
var (
|
||||
gitBranch string
|
||||
gitSince string
|
||||
gitDepth int
|
||||
stagedOnly bool
|
||||
)
|
||||
|
||||
var gitCmd = &cobra.Command{
|
||||
Use: "git [repo-path]",
|
||||
Short: "Scan git repository history for secrets",
|
||||
Args: cobra.MaximumNArgs(1),
|
||||
RunE: runGit,
|
||||
}
|
||||
|
||||
func init() {
|
||||
gitCmd.Flags().StringVarP(
|
||||
&gitBranch, "branch", "b", "",
|
||||
"branch to scan (default: HEAD)",
|
||||
)
|
||||
gitCmd.Flags().StringVar(
|
||||
&gitSince, "since", "",
|
||||
"scan commits since date (YYYY-MM-DD)",
|
||||
)
|
||||
gitCmd.Flags().IntVarP(
|
||||
&gitDepth, "depth", "d", 0,
|
||||
"max number of commits to scan (0 = all)",
|
||||
)
|
||||
gitCmd.Flags().BoolVar(
|
||||
&stagedOnly, "staged", false,
|
||||
"scan only staged changes",
|
||||
)
|
||||
}
|
||||
|
||||
func runGit(cmd *cobra.Command, args []string) error {
|
||||
path := "."
|
||||
if len(args) > 0 {
|
||||
path = args[0]
|
||||
}
|
||||
|
||||
ui.PrintBanner()
|
||||
|
||||
reg := rules.NewRegistry()
|
||||
rules.RegisterBuiltins(reg)
|
||||
applyRuleConfig(reg)
|
||||
|
||||
if gitSince == "" && cfg != nil && cfg.Scan.Since != "" {
|
||||
gitSince = cfg.Scan.Since
|
||||
}
|
||||
if gitDepth == 0 && cfg != nil && cfg.Scan.Depth > 0 {
|
||||
gitDepth = cfg.Scan.Depth
|
||||
}
|
||||
|
||||
src := source.NewGit(
|
||||
path, gitBranch, gitSince, gitDepth,
|
||||
stagedOnly, maxSize, excludes,
|
||||
)
|
||||
|
||||
return executeScan(cmd.Context(), reg, src)
|
||||
}
|
||||
|
|
@ -0,0 +1,74 @@
|
|||
// ©AngelaMos | 2026
|
||||
// init.go
|
||||
|
||||
package cli
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"github.com/CarterPerez-dev/portia/internal/config"
|
||||
"github.com/CarterPerez-dev/portia/internal/ui"
|
||||
)
|
||||
|
||||
var initCmd = &cobra.Command{
|
||||
Use: "init",
|
||||
Short: "Create a default .portia.toml config file",
|
||||
RunE: runInit,
|
||||
}
|
||||
|
||||
var pyprojectCmd = &cobra.Command{
|
||||
Use: "pyproject",
|
||||
Short: "Create a pyproject.toml with [tool.portia] configuration",
|
||||
RunE: runPyproject,
|
||||
}
|
||||
|
||||
func runInit(_ *cobra.Command, _ []string) error {
|
||||
if _, err := os.Stat(config.DefaultConfigFile); err == nil {
|
||||
fmt.Fprintf(os.Stderr, "%s %s already exists\n",
|
||||
ui.Warning, config.DefaultConfigFile)
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := os.WriteFile( //nolint:gosec
|
||||
config.DefaultConfigFile,
|
||||
[]byte(config.DefaultTemplate()),
|
||||
0o644,
|
||||
); err != nil {
|
||||
return fmt.Errorf("write config: %w", err)
|
||||
}
|
||||
|
||||
fmt.Fprintf(os.Stdout, "%s Created %s\n", //nolint:errcheck
|
||||
ui.Check, ui.CyanBold(config.DefaultConfigFile))
|
||||
return nil
|
||||
}
|
||||
|
||||
func runPyproject(_ *cobra.Command, _ []string) error {
|
||||
if _, err := os.Stat("pyproject.toml"); err == nil {
|
||||
return fmt.Errorf("pyproject.toml already exists")
|
||||
}
|
||||
|
||||
dir, err := os.Getwd()
|
||||
if err != nil {
|
||||
return fmt.Errorf("get working directory: %w", err)
|
||||
}
|
||||
|
||||
name := filepath.Base(dir)
|
||||
content := config.PyprojectTemplate(name)
|
||||
|
||||
if err := os.WriteFile( //nolint:gosec
|
||||
"pyproject.toml",
|
||||
[]byte(content),
|
||||
0o644,
|
||||
); err != nil {
|
||||
return fmt.Errorf("write pyproject.toml: %w", err)
|
||||
}
|
||||
|
||||
fmt.Fprintf(os.Stdout, "\n %s %s\n\n", //nolint:errcheck
|
||||
ui.HiGreen(ui.Check),
|
||||
ui.HiGreen("Created pyproject.toml with [tool.portia] configuration"))
|
||||
return nil
|
||||
}
|
||||
|
|
@ -0,0 +1,148 @@
|
|||
// ©AngelaMos | 2026
|
||||
// root.go
|
||||
|
||||
package cli
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
|
||||
"github.com/fatih/color"
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"github.com/CarterPerez-dev/portia/internal/config"
|
||||
"github.com/CarterPerez-dev/portia/internal/ui"
|
||||
)
|
||||
|
||||
var (
|
||||
cfgFile string
|
||||
format string
|
||||
verbose bool
|
||||
noColor bool
|
||||
excludes []string
|
||||
maxSize int64
|
||||
enableHIBP bool
|
||||
)
|
||||
|
||||
var rootCmd = &cobra.Command{
|
||||
Use: "portia",
|
||||
Short: "Secrets scanner for codebases and git repositories",
|
||||
Long: `Portia scans codebases and git history for leaked secrets,
|
||||
API keys, passwords, tokens, and private keys.`,
|
||||
SilenceUsage: true,
|
||||
SilenceErrors: true,
|
||||
}
|
||||
|
||||
func Execute() {
|
||||
if err := run(); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s %s\n",
|
||||
ui.Cross, ui.Red(err.Error()))
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func run() error {
|
||||
ctx, cancel := signal.NotifyContext(
|
||||
rootCmd.Context(), os.Interrupt, syscall.SIGTERM,
|
||||
)
|
||||
defer cancel()
|
||||
|
||||
rootCmd.SetContext(ctx)
|
||||
return rootCmd.ExecuteContext(ctx)
|
||||
}
|
||||
|
||||
func init() {
|
||||
cobra.OnInitialize(initConfig)
|
||||
|
||||
rootCmd.PersistentFlags().StringVar(
|
||||
&cfgFile, "config", "",
|
||||
"config file (default .portia.toml)",
|
||||
)
|
||||
rootCmd.PersistentFlags().StringVarP(
|
||||
&format, "format", "f", "",
|
||||
"output format: terminal, json, sarif",
|
||||
)
|
||||
rootCmd.PersistentFlags().BoolVarP(
|
||||
&verbose, "verbose", "v", false,
|
||||
"verbose output",
|
||||
)
|
||||
rootCmd.PersistentFlags().BoolVar(
|
||||
&noColor, "no-color", false,
|
||||
"disable colored output",
|
||||
)
|
||||
rootCmd.PersistentFlags().StringSliceVarP(
|
||||
&excludes, "exclude", "e", nil,
|
||||
"paths to exclude (repeatable)",
|
||||
)
|
||||
rootCmd.PersistentFlags().Int64Var(
|
||||
&maxSize, "max-size", 0,
|
||||
"max file size in bytes (default 1MB)",
|
||||
)
|
||||
rootCmd.PersistentFlags().BoolVar(
|
||||
&enableHIBP, "hibp", false,
|
||||
"enable HIBP breach verification",
|
||||
)
|
||||
|
||||
rootCmd.AddCommand(scanCmd)
|
||||
rootCmd.AddCommand(gitCmd)
|
||||
rootCmd.AddCommand(initCmd)
|
||||
rootCmd.AddCommand(configCmd)
|
||||
rootCmd.AddCommand(pyprojectCmd)
|
||||
|
||||
defaultHelp := rootCmd.HelpFunc()
|
||||
rootCmd.SetHelpFunc(
|
||||
func(cmd *cobra.Command, args []string) {
|
||||
if cmd.Root() == cmd {
|
||||
ui.PrintBannerWithArt()
|
||||
} else {
|
||||
ui.PrintBanner()
|
||||
}
|
||||
defaultHelp(cmd, args)
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
var cfg *config.Config
|
||||
|
||||
func initConfig() {
|
||||
if noColor {
|
||||
color.NoColor = true
|
||||
}
|
||||
|
||||
loaded, err := config.Load(cfgFile)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s config error: %s\n",
|
||||
ui.Warning, err)
|
||||
}
|
||||
if loaded != nil {
|
||||
cfg = loaded
|
||||
} else {
|
||||
cfg = &config.Config{}
|
||||
}
|
||||
|
||||
if format == "" && cfg.Output.Format != "" {
|
||||
format = cfg.Output.Format
|
||||
}
|
||||
if format == "" {
|
||||
format = "terminal"
|
||||
}
|
||||
|
||||
if !verbose && cfg.Output.Verbose {
|
||||
verbose = true
|
||||
}
|
||||
if !noColor && cfg.Output.NoColor {
|
||||
noColor = true
|
||||
color.NoColor = true
|
||||
}
|
||||
if !enableHIBP && cfg.HIBP.Enabled {
|
||||
enableHIBP = true
|
||||
}
|
||||
if maxSize == 0 && cfg.Scan.MaxFileSize > 0 {
|
||||
maxSize = cfg.Scan.MaxFileSize
|
||||
}
|
||||
if len(excludes) == 0 && len(cfg.Scan.Excludes) > 0 {
|
||||
excludes = cfg.Scan.Excludes
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,119 @@
|
|||
// ©AngelaMos | 2026
|
||||
// scan.go
|
||||
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"github.com/CarterPerez-dev/portia/internal/engine"
|
||||
"github.com/CarterPerez-dev/portia/internal/hibp"
|
||||
"github.com/CarterPerez-dev/portia/internal/reporter"
|
||||
"github.com/CarterPerez-dev/portia/internal/rules"
|
||||
"github.com/CarterPerez-dev/portia/internal/source"
|
||||
"github.com/CarterPerez-dev/portia/internal/ui"
|
||||
"github.com/CarterPerez-dev/portia/pkg/types"
|
||||
)
|
||||
|
||||
var scanCmd = &cobra.Command{
|
||||
Use: "scan [path]",
|
||||
Short: "Scan a directory for secrets",
|
||||
Args: cobra.MaximumNArgs(1),
|
||||
RunE: runScan,
|
||||
}
|
||||
|
||||
func runScan(cmd *cobra.Command, args []string) error {
|
||||
path := "."
|
||||
if len(args) > 0 {
|
||||
path = args[0]
|
||||
}
|
||||
|
||||
ui.PrintBanner()
|
||||
|
||||
reg := rules.NewRegistry()
|
||||
rules.RegisterBuiltins(reg)
|
||||
applyRuleConfig(reg)
|
||||
|
||||
src := source.NewDirectory(path, maxSize, excludes)
|
||||
|
||||
return executeScan(cmd.Context(), reg, src)
|
||||
}
|
||||
|
||||
func executeScan(
|
||||
ctx context.Context,
|
||||
reg *rules.Registry,
|
||||
src source.Source,
|
||||
) error {
|
||||
spin := ui.NewSpinner("Scanning for secrets...")
|
||||
if !verbose {
|
||||
spin.Start()
|
||||
}
|
||||
|
||||
start := time.Now()
|
||||
p := engine.NewPipeline(reg)
|
||||
p.SetVerbose(verbose)
|
||||
result, err := p.Run(ctx, src)
|
||||
spin.Stop()
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
result.Duration = time.Since(start)
|
||||
|
||||
if enableHIBP && len(result.Findings) > 0 {
|
||||
checkHIBP(ctx, result)
|
||||
}
|
||||
|
||||
rep := reporter.New(format)
|
||||
return rep.Report(os.Stdout, result)
|
||||
}
|
||||
|
||||
func checkHIBP(ctx context.Context, result *types.ScanResult) {
|
||||
client := hibp.NewClient()
|
||||
spin := ui.NewSpinner("Checking HIBP breach database...")
|
||||
spin.Start()
|
||||
defer spin.Stop()
|
||||
|
||||
for i := range result.Findings {
|
||||
f := &result.Findings[i]
|
||||
if f.Secret == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
if f.RuleID != "generic-password" &&
|
||||
f.RuleID != "generic-secret" {
|
||||
f.HIBPStatus = types.HIBPSkipped
|
||||
continue
|
||||
}
|
||||
|
||||
res, err := client.Check(ctx, f.Secret)
|
||||
result.HIBPChecked++
|
||||
|
||||
if err != nil {
|
||||
f.HIBPStatus = types.HIBPError
|
||||
continue
|
||||
}
|
||||
|
||||
if res.Breached {
|
||||
f.HIBPStatus = types.HIBPBreached
|
||||
f.BreachCount = res.Count
|
||||
result.HIBPBreached++
|
||||
} else {
|
||||
f.HIBPStatus = types.HIBPClean
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func applyRuleConfig(reg *rules.Registry) {
|
||||
if cfg == nil {
|
||||
return
|
||||
}
|
||||
if len(cfg.Rules.Disable) > 0 {
|
||||
reg.Disable(cfg.Rules.Disable...)
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,202 @@
|
|||
// ©AngelaMos | 2026
|
||||
// config.go
|
||||
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/pelletier/go-toml/v2"
|
||||
)
|
||||
|
||||
const DefaultConfigFile = ".portia.toml"
|
||||
|
||||
type Config struct {
|
||||
Rules RulesConfig `toml:"rules"`
|
||||
Scan ScanConfig `toml:"scan"`
|
||||
Output OutputConfig `toml:"output"`
|
||||
HIBP HIBPConfig `toml:"hibp"`
|
||||
Allowlist AllowlistConfig `toml:"allowlist"`
|
||||
}
|
||||
|
||||
type RulesConfig struct {
|
||||
Disable []string `toml:"disable"`
|
||||
Enable []string `toml:"enable"`
|
||||
}
|
||||
|
||||
type ScanConfig struct {
|
||||
MaxFileSize int64 `toml:"max_file_size"`
|
||||
Excludes []string `toml:"excludes"`
|
||||
Depth int `toml:"depth"`
|
||||
Since string `toml:"since"`
|
||||
}
|
||||
|
||||
type OutputConfig struct {
|
||||
Format string `toml:"format"`
|
||||
Verbose bool `toml:"verbose"`
|
||||
NoColor bool `toml:"no_color"`
|
||||
}
|
||||
|
||||
type HIBPConfig struct {
|
||||
Enabled bool `toml:"enabled"`
|
||||
}
|
||||
|
||||
type AllowlistConfig struct {
|
||||
Paths []string `toml:"paths"`
|
||||
Values []string `toml:"values"`
|
||||
Stopwords []string `toml:"stopwords"`
|
||||
}
|
||||
|
||||
func Load(path string) (*Config, error) {
|
||||
cfg := &Config{
|
||||
Scan: ScanConfig{
|
||||
MaxFileSize: 1 << 20,
|
||||
},
|
||||
Output: OutputConfig{
|
||||
Format: "terminal",
|
||||
},
|
||||
}
|
||||
|
||||
if path == "" {
|
||||
path = findConfig()
|
||||
}
|
||||
|
||||
if path == "" {
|
||||
if pyCfg, ok := loadFromPyproject(); ok {
|
||||
return pyCfg, nil
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(path) //nolint:gosec
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return cfg, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := toml.Unmarshal(data, cfg); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
type pyprojectWrapper struct {
|
||||
Tool struct {
|
||||
Portia Config `toml:"portia"`
|
||||
} `toml:"tool"`
|
||||
}
|
||||
|
||||
func loadFromPyproject() (*Config, bool) {
|
||||
data, err := os.ReadFile("pyproject.toml")
|
||||
if err != nil {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
var wrapper pyprojectWrapper
|
||||
if err := toml.Unmarshal(data, &wrapper); err != nil {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
cfg := wrapper.Tool.Portia
|
||||
if cfg.Output.Format == "" && !cfg.Output.Verbose &&
|
||||
!cfg.Output.NoColor && !cfg.HIBP.Enabled &&
|
||||
len(cfg.Rules.Disable) == 0 && len(cfg.Scan.Excludes) == 0 {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
if cfg.Scan.MaxFileSize == 0 {
|
||||
cfg.Scan.MaxFileSize = 1 << 20
|
||||
}
|
||||
if cfg.Output.Format == "" {
|
||||
cfg.Output.Format = "terminal"
|
||||
}
|
||||
|
||||
return &cfg, true
|
||||
}
|
||||
|
||||
func findConfig() string {
|
||||
candidates := []string{
|
||||
DefaultConfigFile,
|
||||
filepath.Join(".portia", "config.toml"),
|
||||
}
|
||||
|
||||
for _, c := range candidates {
|
||||
if _, err := os.Stat(c); err == nil {
|
||||
return c
|
||||
}
|
||||
}
|
||||
|
||||
home, err := os.UserHomeDir()
|
||||
if err == nil {
|
||||
global := filepath.Join(home, ".config", "portia", "config.toml")
|
||||
if _, err := os.Stat(global); err == nil {
|
||||
return global
|
||||
}
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
|
||||
func PyprojectTemplate(name string) string {
|
||||
return fmt.Sprintf(`[project]
|
||||
name = "%s"
|
||||
version = "0.1.0"
|
||||
description = ""
|
||||
requires-python = ">=3.13"
|
||||
dependencies = []
|
||||
|
||||
[tool.portia]
|
||||
# [tool.portia.rules]
|
||||
# disable = ["generic-password"]
|
||||
|
||||
# [tool.portia.scan]
|
||||
# max_file_size = 1048576
|
||||
# excludes = ["*.test.js", "fixtures/"]
|
||||
|
||||
# [tool.portia.output]
|
||||
# format = "terminal"
|
||||
# verbose = false
|
||||
# no_color = false
|
||||
|
||||
# [tool.portia.hibp]
|
||||
# enabled = false
|
||||
|
||||
# [tool.portia.allowlist]
|
||||
# paths = ["test/fixtures/"]
|
||||
# values = ["my_test_value"]
|
||||
# stopwords = ["mycompany"]
|
||||
`, name)
|
||||
}
|
||||
|
||||
func DefaultTemplate() string {
|
||||
return `# ©AngelaMos | 2026
|
||||
# .portia.toml
|
||||
|
||||
[rules]
|
||||
# disable = ["generic-password"]
|
||||
|
||||
[scan]
|
||||
max_file_size = 1048576
|
||||
# excludes = ["*.test.js", "fixtures/"]
|
||||
# depth = 0
|
||||
# since = ""
|
||||
|
||||
[output]
|
||||
format = "terminal"
|
||||
verbose = false
|
||||
no_color = false
|
||||
|
||||
[hibp]
|
||||
enabled = false
|
||||
|
||||
[allowlist]
|
||||
# paths = ["test/fixtures/"]
|
||||
# values = ["my_test_value"]
|
||||
# stopwords = ["mycompany"]
|
||||
`
|
||||
}
|
||||
|
|
@ -0,0 +1,173 @@
|
|||
// ©AngelaMos | 2026
|
||||
// config_test.go
|
||||
|
||||
package config
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestLoadExplicitPath(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "portia.toml")
|
||||
|
||||
content := `
|
||||
[rules]
|
||||
disable = ["generic-password"]
|
||||
|
||||
[scan]
|
||||
max_file_size = 2097152
|
||||
excludes = ["vendor/", "*.test.js"]
|
||||
|
||||
[output]
|
||||
format = "json"
|
||||
verbose = true
|
||||
|
||||
[hibp]
|
||||
enabled = true
|
||||
`
|
||||
require.NoError(t, os.WriteFile(path, []byte(content), 0o644))
|
||||
|
||||
cfg, err := Load(path)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, cfg)
|
||||
|
||||
assert.Equal(t, []string{"generic-password"}, cfg.Rules.Disable)
|
||||
assert.Equal(t, int64(2097152), cfg.Scan.MaxFileSize)
|
||||
assert.Equal(t, []string{"vendor/", "*.test.js"}, cfg.Scan.Excludes)
|
||||
assert.Equal(t, "json", cfg.Output.Format)
|
||||
assert.True(t, cfg.Output.Verbose)
|
||||
assert.True(t, cfg.HIBP.Enabled)
|
||||
}
|
||||
|
||||
func TestLoadNonexistentExplicitPath(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cfg, err := Load("/nonexistent/path/portia.toml")
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, cfg)
|
||||
assert.Equal(t, "terminal", cfg.Output.Format)
|
||||
assert.Equal(t, int64(1<<20), cfg.Scan.MaxFileSize)
|
||||
}
|
||||
|
||||
func TestLoadEmptyPathReturnsDefaults(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
origDir, err := os.Getwd()
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, os.Chdir(dir))
|
||||
defer os.Chdir(origDir) //nolint:errcheck
|
||||
|
||||
cfg, err := Load("")
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, cfg)
|
||||
assert.Equal(t, "terminal", cfg.Output.Format)
|
||||
assert.Equal(t, int64(1<<20), cfg.Scan.MaxFileSize)
|
||||
}
|
||||
|
||||
func TestLoadDiscoveryFromCurrentDir(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
origDir, err := os.Getwd()
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, os.Chdir(dir))
|
||||
defer os.Chdir(origDir) //nolint:errcheck
|
||||
|
||||
content := `
|
||||
[output]
|
||||
format = "sarif"
|
||||
`
|
||||
require.NoError(t, os.WriteFile(
|
||||
filepath.Join(dir, DefaultConfigFile),
|
||||
[]byte(content), 0o644,
|
||||
))
|
||||
|
||||
cfg, err := Load("")
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, cfg)
|
||||
assert.Equal(t, "sarif", cfg.Output.Format)
|
||||
}
|
||||
|
||||
func TestLoadPyprojectFallback(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
origDir, err := os.Getwd()
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, os.Chdir(dir))
|
||||
defer os.Chdir(origDir) //nolint:errcheck
|
||||
|
||||
content := `
|
||||
[project]
|
||||
name = "myproject"
|
||||
|
||||
[tool.portia.output]
|
||||
format = "json"
|
||||
verbose = true
|
||||
`
|
||||
require.NoError(t, os.WriteFile(
|
||||
filepath.Join(dir, "pyproject.toml"),
|
||||
[]byte(content), 0o644,
|
||||
))
|
||||
|
||||
cfg, err := Load("")
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, cfg)
|
||||
assert.Equal(t, "json", cfg.Output.Format)
|
||||
assert.True(t, cfg.Output.Verbose)
|
||||
}
|
||||
|
||||
func TestLoadPyprojectIgnoredWhenEmpty(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
origDir, err := os.Getwd()
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, os.Chdir(dir))
|
||||
defer os.Chdir(origDir) //nolint:errcheck
|
||||
|
||||
content := `
|
||||
[project]
|
||||
name = "myproject"
|
||||
`
|
||||
require.NoError(t, os.WriteFile(
|
||||
filepath.Join(dir, "pyproject.toml"),
|
||||
[]byte(content), 0o644,
|
||||
))
|
||||
|
||||
cfg, err := Load("")
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, cfg)
|
||||
assert.Equal(t, "terminal", cfg.Output.Format)
|
||||
}
|
||||
|
||||
func TestDefaultTemplateValidTOML(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tmpl := DefaultTemplate()
|
||||
assert.Contains(t, tmpl, "[rules]")
|
||||
assert.Contains(t, tmpl, "[scan]")
|
||||
assert.Contains(t, tmpl, "[output]")
|
||||
assert.Contains(t, tmpl, "[hibp]")
|
||||
assert.Contains(t, tmpl, "[allowlist]")
|
||||
}
|
||||
|
||||
func TestPyprojectTemplateIncludesName(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tmpl := PyprojectTemplate("myapp")
|
||||
assert.Contains(t, tmpl, `name = "myapp"`)
|
||||
assert.Contains(t, tmpl, "[tool.portia]")
|
||||
}
|
||||
|
||||
func TestLoadInvalidTOML(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "bad.toml")
|
||||
require.NoError(t, os.WriteFile(path, []byte(`[invalid`), 0o644))
|
||||
|
||||
_, err := Load(path)
|
||||
assert.Error(t, err)
|
||||
}
|
||||
|
|
@ -0,0 +1,187 @@
|
|||
// ©AngelaMos | 2026
|
||||
// detector.go
|
||||
|
||||
package engine
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/CarterPerez-dev/portia/internal/rules"
|
||||
"github.com/CarterPerez-dev/portia/pkg/types"
|
||||
)
|
||||
|
||||
type Detector struct {
|
||||
registry *rules.Registry
|
||||
}
|
||||
|
||||
func NewDetector(reg *rules.Registry) *Detector {
|
||||
return &Detector{registry: reg}
|
||||
}
|
||||
|
||||
func (d *Detector) Detect(chunk types.Chunk) []types.Finding { //nolint:gocognit
|
||||
lines := strings.Split(chunk.Content, "\n")
|
||||
var findings []types.Finding
|
||||
|
||||
matched := d.registry.MatchKeywords(chunk.Content)
|
||||
for _, rule := range matched {
|
||||
for i, line := range lines {
|
||||
matches := rule.Pattern.FindAllStringSubmatchIndex(
|
||||
line, -1,
|
||||
)
|
||||
if len(matches) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
for _, loc := range matches {
|
||||
secret := extractSecret(line, loc, rule.SecretGroup)
|
||||
if secret == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
match := line[loc[0]:loc[1]]
|
||||
|
||||
finding := types.Finding{
|
||||
RuleID: rule.ID,
|
||||
Description: rule.Description,
|
||||
Severity: rule.Severity,
|
||||
Match: match,
|
||||
Secret: secret,
|
||||
FilePath: chunk.FilePath,
|
||||
LineNumber: chunk.LineStart + i,
|
||||
LineContent: line,
|
||||
CommitSHA: chunk.CommitSHA,
|
||||
Author: chunk.Author,
|
||||
CommitDate: chunk.CommitDate,
|
||||
}
|
||||
|
||||
if rule.Entropy != nil {
|
||||
charset := rules.DetectCharset(secret)
|
||||
var charsetStr string
|
||||
switch charset {
|
||||
case "hex":
|
||||
charsetStr = rules.HexCharset
|
||||
case "base64":
|
||||
charsetStr = rules.Base64Charset
|
||||
default:
|
||||
charsetStr = rules.AlphanumericCharset
|
||||
}
|
||||
entropy := rules.ShannonEntropy(
|
||||
secret, charsetStr,
|
||||
)
|
||||
finding.Entropy = entropy
|
||||
|
||||
if entropy < *rule.Entropy {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if !FilterFinding(&finding, rule) {
|
||||
continue
|
||||
}
|
||||
|
||||
findings = append(findings, finding)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
findings = append(
|
||||
findings,
|
||||
d.detectHighEntropy(chunk, lines, findings)...,
|
||||
)
|
||||
|
||||
return findings
|
||||
}
|
||||
|
||||
var entropyCharsets = []struct {
|
||||
charset string
|
||||
threshold float64
|
||||
}{
|
||||
{rules.Base64Charset, 4.5},
|
||||
{rules.HexCharset, 3.5},
|
||||
}
|
||||
|
||||
func (d *Detector) detectHighEntropy(
|
||||
chunk types.Chunk,
|
||||
lines []string,
|
||||
existing []types.Finding,
|
||||
) []types.Finding {
|
||||
var findings []types.Finding
|
||||
dummyRule := &types.Rule{}
|
||||
|
||||
for i, line := range lines {
|
||||
if !HasAssignmentOperator(line) {
|
||||
continue
|
||||
}
|
||||
lineNum := chunk.LineStart + i
|
||||
|
||||
for _, cs := range entropyCharsets {
|
||||
tokens := rules.ExtractHighEntropyTokens(
|
||||
line, cs.charset, cs.threshold, 20,
|
||||
)
|
||||
for _, token := range tokens {
|
||||
if isAlreadyCaught(
|
||||
token.Value, lineNum, existing,
|
||||
) {
|
||||
continue
|
||||
}
|
||||
if isAlreadyCaught(
|
||||
token.Value, lineNum, findings,
|
||||
) {
|
||||
continue
|
||||
}
|
||||
|
||||
finding := types.Finding{
|
||||
RuleID: "high-entropy-string",
|
||||
Description: "High Entropy String",
|
||||
Severity: types.SeverityMedium,
|
||||
Match: token.Value,
|
||||
Secret: token.Value,
|
||||
Entropy: token.Entropy,
|
||||
FilePath: chunk.FilePath,
|
||||
LineNumber: lineNum,
|
||||
LineContent: line,
|
||||
CommitSHA: chunk.CommitSHA,
|
||||
Author: chunk.Author,
|
||||
CommitDate: chunk.CommitDate,
|
||||
}
|
||||
|
||||
if !FilterFinding(&finding, dummyRule) {
|
||||
continue
|
||||
}
|
||||
|
||||
findings = append(findings, finding)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return findings
|
||||
}
|
||||
|
||||
func isAlreadyCaught(
|
||||
token string, lineNum int, existing []types.Finding,
|
||||
) bool {
|
||||
for _, f := range existing {
|
||||
if f.LineNumber == lineNum &&
|
||||
(strings.Contains(f.Secret, token) ||
|
||||
strings.Contains(token, f.Secret)) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func extractSecret(
|
||||
line string, loc []int, group int,
|
||||
) string {
|
||||
if group > 0 && len(loc) > group*2+1 {
|
||||
start := loc[group*2]
|
||||
end := loc[group*2+1]
|
||||
if start >= 0 && end >= 0 {
|
||||
return line[start:end]
|
||||
}
|
||||
}
|
||||
if len(loc) >= 2 {
|
||||
return line[loc[0]:loc[1]]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
|
@ -0,0 +1,259 @@
|
|||
// ©AngelaMos | 2026
|
||||
// detector_test.go
|
||||
|
||||
package engine
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/CarterPerez-dev/portia/internal/rules"
|
||||
"github.com/CarterPerez-dev/portia/pkg/types"
|
||||
)
|
||||
|
||||
func testRegistry() *rules.Registry {
|
||||
reg := rules.NewRegistry()
|
||||
reg.Register(&types.Rule{
|
||||
ID: "test-aws-key",
|
||||
Description: "AWS Access Key ID",
|
||||
Severity: types.SeverityCritical,
|
||||
Keywords: []string{"AKIA"},
|
||||
Pattern: regexp.MustCompile(
|
||||
`\b((?:AKIA)[0-9A-Z]{16})\b`,
|
||||
),
|
||||
SecretGroup: 1,
|
||||
SecretType: types.SecretTypeAPIKey,
|
||||
})
|
||||
reg.Register(&types.Rule{
|
||||
ID: "test-generic-password",
|
||||
Description: "Generic Password",
|
||||
Severity: types.SeverityHigh,
|
||||
Keywords: []string{"password"},
|
||||
Pattern: regexp.MustCompile(
|
||||
`(?i)password\s*[:=]\s*['"]([^'"]{8,})['"]`,
|
||||
),
|
||||
SecretGroup: 1,
|
||||
Entropy: func() *float64 { f := 3.0; return &f }(),
|
||||
SecretType: types.SecretTypePassword,
|
||||
})
|
||||
reg.Register(&types.Rule{
|
||||
ID: "test-stripe-key",
|
||||
Description: "Stripe Live Key",
|
||||
Severity: types.SeverityCritical,
|
||||
Keywords: []string{"sk_live_"},
|
||||
Pattern: regexp.MustCompile(
|
||||
`\b(sk_live_[a-zA-Z0-9]{24,})\b`,
|
||||
),
|
||||
SecretGroup: 1,
|
||||
SecretType: types.SecretTypeAPIKey,
|
||||
})
|
||||
return reg
|
||||
}
|
||||
|
||||
func TestDetectAWSKey(t *testing.T) {
|
||||
t.Parallel()
|
||||
d := NewDetector(testRegistry())
|
||||
|
||||
chunk := types.Chunk{ //nolint:gosec
|
||||
Content: `aws_key = "AKIAIOSFODNN7EXAMPLE"`,
|
||||
FilePath: "config.py",
|
||||
LineStart: 1,
|
||||
}
|
||||
|
||||
findings := d.Detect(chunk)
|
||||
require.Len(t, findings, 1)
|
||||
assert.Equal(t, "test-aws-key", findings[0].RuleID)
|
||||
assert.Equal(t, "AKIAIOSFODNN7EXAMPLE", findings[0].Secret) //nolint:gosec
|
||||
assert.Equal(t, "config.py", findings[0].FilePath)
|
||||
assert.Equal(t, 1, findings[0].LineNumber)
|
||||
assert.Equal(t, types.SeverityCritical, findings[0].Severity)
|
||||
}
|
||||
|
||||
func TestDetectStripeKey(t *testing.T) {
|
||||
t.Parallel()
|
||||
d := NewDetector(testRegistry())
|
||||
|
||||
chunk := types.Chunk{ //nolint:gosec
|
||||
Content: `STRIPE_KEY=sk_live_4eC39HqLyjWDarjtT1zdp7dc`,
|
||||
FilePath: "env.sh",
|
||||
LineStart: 10,
|
||||
}
|
||||
|
||||
findings := d.Detect(chunk)
|
||||
require.Len(t, findings, 1)
|
||||
assert.Equal(t, "test-stripe-key", findings[0].RuleID)
|
||||
assert.Equal(t,
|
||||
"sk_live_4eC39HqLyjWDarjtT1zdp7dc",
|
||||
findings[0].Secret,
|
||||
)
|
||||
}
|
||||
|
||||
func TestDetectPasswordWithEntropy(t *testing.T) {
|
||||
t.Parallel()
|
||||
d := NewDetector(testRegistry())
|
||||
|
||||
chunk := types.Chunk{
|
||||
Content: `password = "xK9mP2vL5nQ8jR3t"`,
|
||||
FilePath: "config.ini",
|
||||
LineStart: 5,
|
||||
}
|
||||
|
||||
findings := d.Detect(chunk)
|
||||
require.Len(t, findings, 1)
|
||||
assert.Equal(t, "test-generic-password", findings[0].RuleID)
|
||||
assert.Greater(t, findings[0].Entropy, 3.0)
|
||||
}
|
||||
|
||||
func TestDetectPasswordLowEntropyFiltered(t *testing.T) {
|
||||
t.Parallel()
|
||||
d := NewDetector(testRegistry())
|
||||
|
||||
chunk := types.Chunk{
|
||||
Content: `password = "aaaaaaaaaa"`,
|
||||
FilePath: "config.ini",
|
||||
LineStart: 1,
|
||||
}
|
||||
|
||||
findings := d.Detect(chunk)
|
||||
assert.Empty(t, findings)
|
||||
}
|
||||
|
||||
func TestDetectNoKeywords(t *testing.T) {
|
||||
t.Parallel()
|
||||
d := NewDetector(testRegistry())
|
||||
|
||||
chunk := types.Chunk{
|
||||
Content: "just some normal code here",
|
||||
FilePath: "main.go",
|
||||
LineStart: 1,
|
||||
}
|
||||
|
||||
findings := d.Detect(chunk)
|
||||
assert.Empty(t, findings)
|
||||
}
|
||||
|
||||
func TestDetectPlaceholderFiltered(t *testing.T) {
|
||||
t.Parallel()
|
||||
d := NewDetector(testRegistry())
|
||||
|
||||
chunk := types.Chunk{
|
||||
Content: `password = "your-password-here"`,
|
||||
FilePath: "config.py",
|
||||
LineStart: 1,
|
||||
}
|
||||
|
||||
findings := d.Detect(chunk)
|
||||
assert.Empty(t, findings)
|
||||
}
|
||||
|
||||
func TestDetectMultipleFindings(t *testing.T) {
|
||||
t.Parallel()
|
||||
d := NewDetector(testRegistry())
|
||||
|
||||
chunk := types.Chunk{
|
||||
Content: `aws_key = "AKIAIOSFODNN7EXAMPLE"` + "\n" +
|
||||
`stripe = sk_live_4eC39HqLyjWDarjtT1zdp7dc`,
|
||||
FilePath: "config.py",
|
||||
LineStart: 1,
|
||||
}
|
||||
|
||||
findings := d.Detect(chunk)
|
||||
require.Len(t, findings, 2)
|
||||
|
||||
ruleIDs := map[string]bool{}
|
||||
for _, f := range findings {
|
||||
ruleIDs[f.RuleID] = true
|
||||
}
|
||||
assert.True(t, ruleIDs["test-aws-key"])
|
||||
assert.True(t, ruleIDs["test-stripe-key"])
|
||||
}
|
||||
|
||||
func TestDetectLineNumbers(t *testing.T) {
|
||||
t.Parallel()
|
||||
d := NewDetector(testRegistry())
|
||||
|
||||
chunk := types.Chunk{
|
||||
Content: "line 1\n" +
|
||||
"line 2\n" +
|
||||
`AKIAIOSFODNN7EXAMPLE` + "\n" +
|
||||
"line 4\n",
|
||||
FilePath: "data.txt",
|
||||
LineStart: 10,
|
||||
}
|
||||
|
||||
findings := d.Detect(chunk)
|
||||
require.Len(t, findings, 1)
|
||||
assert.Equal(t, 12, findings[0].LineNumber)
|
||||
}
|
||||
|
||||
func TestDetectAllowedPathFiltered(t *testing.T) {
|
||||
t.Parallel()
|
||||
d := NewDetector(testRegistry())
|
||||
|
||||
chunk := types.Chunk{ //nolint:gosec
|
||||
Content: `AKIAIOSFODNN7EXAMPLE`,
|
||||
FilePath: "go.sum",
|
||||
LineStart: 1,
|
||||
}
|
||||
|
||||
findings := d.Detect(chunk)
|
||||
assert.Empty(t, findings)
|
||||
}
|
||||
|
||||
func TestExtractSecret(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
line := `key = "AKIAIOSFODNN7EXAMPLE"` //nolint:gosec
|
||||
loc := []int{7, 27, 7, 27}
|
||||
got := extractSecret(line, loc, 1)
|
||||
assert.Equal(t, "AKIAIOSFODNN7EXAMPLE", got)
|
||||
|
||||
got = extractSecret(line, loc, 0)
|
||||
assert.Equal(t, `AKIAIOSFODNN7EXAMPLE`, got)
|
||||
|
||||
got = extractSecret(line, []int{}, 1)
|
||||
assert.Empty(t, got)
|
||||
}
|
||||
|
||||
func TestDetectHighEntropy(t *testing.T) {
|
||||
t.Parallel()
|
||||
d := NewDetector(testRegistry())
|
||||
|
||||
chunk := types.Chunk{
|
||||
Content: `secret = "aB1cD2eF3gH4iJ5kL6mN7oP8qR9sT0u"`,
|
||||
FilePath: "config.py",
|
||||
LineStart: 1,
|
||||
}
|
||||
|
||||
findings := d.Detect(chunk)
|
||||
require.Len(t, findings, 1)
|
||||
assert.Equal(t, "high-entropy-string", findings[0].RuleID)
|
||||
assert.Equal(t, types.SeverityMedium, findings[0].Severity)
|
||||
assert.Greater(t, findings[0].Entropy, 4.0)
|
||||
}
|
||||
|
||||
func TestDetectHighEntropyNotDuplicated(t *testing.T) {
|
||||
t.Parallel()
|
||||
d := NewDetector(testRegistry())
|
||||
|
||||
chunk := types.Chunk{ //nolint:gosec
|
||||
Content: `aws_key = "AKIAIOSFODNN7EXAMPLE"`,
|
||||
FilePath: "config.py",
|
||||
LineStart: 1,
|
||||
}
|
||||
|
||||
findings := d.Detect(chunk)
|
||||
ruleIDs := map[string]bool{}
|
||||
for _, f := range findings {
|
||||
ruleIDs[f.RuleID] = true
|
||||
}
|
||||
assert.True(t, ruleIDs["test-aws-key"])
|
||||
assert.False(
|
||||
t,
|
||||
ruleIDs["high-entropy-string"],
|
||||
"entropy detector should not duplicate rule findings",
|
||||
)
|
||||
}
|
||||
|
|
@ -0,0 +1,847 @@
|
|||
// ©AngelaMos | 2026
|
||||
// filter.go
|
||||
|
||||
package engine
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/CarterPerez-dev/portia/internal/rules"
|
||||
"github.com/CarterPerez-dev/portia/pkg/types"
|
||||
)
|
||||
|
||||
var templatePatterns = []*regexp.Regexp{
|
||||
regexp.MustCompile(`^\$\{.+\}$`),
|
||||
regexp.MustCompile(`^\{\{.+\}\}$`),
|
||||
regexp.MustCompile(`(?i)os\.(?:getenv|environ)\s*\(`),
|
||||
regexp.MustCompile(`(?i)process\.env\.`),
|
||||
regexp.MustCompile(`(?i)System\.getenv\s*\(`),
|
||||
regexp.MustCompile(`(?i)ENV\[`),
|
||||
regexp.MustCompile(`(?i)env\(\s*['"]`),
|
||||
regexp.MustCompile(`(?i)viper\.Get`),
|
||||
regexp.MustCompile(`(?i)config\.get\s*\(`),
|
||||
regexp.MustCompile(`(?i)config\[`),
|
||||
regexp.MustCompile(`@Value\s*\(\s*"\$\{`),
|
||||
regexp.MustCompile(`<%=\s*ENV\[`),
|
||||
regexp.MustCompile(`\{\{\s*\.Values\.`),
|
||||
}
|
||||
|
||||
var assignmentPattern = regexp.MustCompile(
|
||||
`(?:=|:|=>|:=|\|\||,)\s*['"\x60]?\s*\S`,
|
||||
)
|
||||
|
||||
func IsStopword(
|
||||
secret string, extra []string,
|
||||
) bool {
|
||||
lower := strings.ToLower(secret)
|
||||
if _, ok := stopwords[lower]; ok {
|
||||
return true
|
||||
}
|
||||
|
||||
for _, sw := range extra {
|
||||
if len(sw) >= 4 &&
|
||||
strings.Contains(lower, strings.ToLower(sw)) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
parts := strings.FieldsFunc(lower, func(r rune) bool {
|
||||
return r == '_' || r == '-' || r == '.' || r == '/'
|
||||
})
|
||||
for _, part := range parts {
|
||||
if _, ok := stopwords[part]; ok {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func IsPlaceholder(secret string) bool {
|
||||
for _, re := range rules.GlobalValueAllowlist {
|
||||
if re.MatchString(secret) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func IsTemplated(secret string) bool {
|
||||
for _, re := range templatePatterns {
|
||||
if re.MatchString(secret) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func HasAssignmentOperator(line string) bool {
|
||||
return assignmentPattern.MatchString(line)
|
||||
}
|
||||
|
||||
func IsAllowedPath(
|
||||
path string, allowlist []*regexp.Regexp,
|
||||
) bool {
|
||||
for _, re := range allowlist {
|
||||
if re.MatchString(path) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func FilterFinding(
|
||||
finding *types.Finding,
|
||||
rule *types.Rule,
|
||||
) bool {
|
||||
if IsPlaceholder(finding.Secret) {
|
||||
return false
|
||||
}
|
||||
|
||||
if IsTemplated(finding.Secret) {
|
||||
return false
|
||||
}
|
||||
|
||||
if IsStopword(finding.Secret, rule.Allowlist.Stopwords) {
|
||||
return false
|
||||
}
|
||||
|
||||
for _, re := range rule.Allowlist.Values {
|
||||
if re.MatchString(finding.Secret) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
if IsAllowedPath(
|
||||
finding.FilePath,
|
||||
rules.GlobalPathAllowlist,
|
||||
) {
|
||||
return false
|
||||
}
|
||||
|
||||
for _, re := range rule.Allowlist.Paths {
|
||||
if re.MatchString(finding.FilePath) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
var stopwords = map[string]bool{
|
||||
"abstract": true,
|
||||
"account": true,
|
||||
"action": true,
|
||||
"active": true,
|
||||
"activity": true,
|
||||
"adapter": true,
|
||||
"admin": true,
|
||||
"after": true,
|
||||
"agent": true,
|
||||
"algorithm": true,
|
||||
"alias": true,
|
||||
"allow": true,
|
||||
"alpha": true,
|
||||
"analytics": true,
|
||||
"android": true,
|
||||
"angular": true,
|
||||
"annotation": true,
|
||||
"anonymous": true,
|
||||
"apache": true,
|
||||
"application": true,
|
||||
"archive": true,
|
||||
"array": true,
|
||||
"article": true,
|
||||
"asset": true,
|
||||
"assign": true,
|
||||
"async": true,
|
||||
"attribute": true,
|
||||
"audit": true,
|
||||
"authenticate": true,
|
||||
"author": true,
|
||||
"automate": true,
|
||||
"avatar": true,
|
||||
"backend": true,
|
||||
"backup": true,
|
||||
"badge": true,
|
||||
"balance": true,
|
||||
"banner": true,
|
||||
"baseline": true,
|
||||
"batch": true,
|
||||
"before": true,
|
||||
"benchmark": true,
|
||||
"binary": true,
|
||||
"binding": true,
|
||||
"block": true,
|
||||
"board": true,
|
||||
"body": true,
|
||||
"boolean": true,
|
||||
"border": true,
|
||||
"branch": true,
|
||||
"bridge": true,
|
||||
"broadcast": true,
|
||||
"browser": true,
|
||||
"bucket": true,
|
||||
"buffer": true,
|
||||
"build": true,
|
||||
"builder": true,
|
||||
"bundle": true,
|
||||
"button": true,
|
||||
"cache": true,
|
||||
"calendar": true,
|
||||
"callback": true,
|
||||
"campaign": true,
|
||||
"cancel": true,
|
||||
"candidate": true,
|
||||
"canvas": true,
|
||||
"capture": true,
|
||||
"catalog": true,
|
||||
"category": true,
|
||||
"certificate": true,
|
||||
"chain": true,
|
||||
"channel": true,
|
||||
"chapter": true,
|
||||
"character": true,
|
||||
"chart": true,
|
||||
"check": true,
|
||||
"checkpoint": true,
|
||||
"child": true,
|
||||
"chrome": true,
|
||||
"cipher": true,
|
||||
"circuit": true,
|
||||
"class": true,
|
||||
"clean": true,
|
||||
"clear": true,
|
||||
"click": true,
|
||||
"client": true,
|
||||
"clipboard": true,
|
||||
"clone": true,
|
||||
"close": true,
|
||||
"cloud": true,
|
||||
"cluster": true,
|
||||
"codec": true,
|
||||
"collection": true,
|
||||
"color": true,
|
||||
"column": true,
|
||||
"command": true,
|
||||
"comment": true,
|
||||
"commit": true,
|
||||
"common": true,
|
||||
"community": true,
|
||||
"compact": true,
|
||||
"compare": true,
|
||||
"compatibility": true,
|
||||
"compile": true,
|
||||
"complete": true,
|
||||
"component": true,
|
||||
"compose": true,
|
||||
"compress": true,
|
||||
"compute": true,
|
||||
"condition": true,
|
||||
"config": true,
|
||||
"configure": true,
|
||||
"confirm": true,
|
||||
"connect": true,
|
||||
"connection": true,
|
||||
"connector": true,
|
||||
"console": true,
|
||||
"constant": true,
|
||||
"constraint": true,
|
||||
"construct": true,
|
||||
"consumer": true,
|
||||
"contact": true,
|
||||
"container": true,
|
||||
"content": true,
|
||||
"context": true,
|
||||
"continue": true,
|
||||
"contract": true,
|
||||
"control": true,
|
||||
"controller": true,
|
||||
"convert": true,
|
||||
"cookie": true,
|
||||
"coordinate": true,
|
||||
"copy": true,
|
||||
"counter": true,
|
||||
"coverage": true,
|
||||
"create": true,
|
||||
"credential": true,
|
||||
"criteria": true,
|
||||
"currency": true,
|
||||
"current": true,
|
||||
"cursor": true,
|
||||
"custom": true,
|
||||
"customer": true,
|
||||
"daemon": true,
|
||||
"dashboard": true,
|
||||
"data": true,
|
||||
"database": true,
|
||||
"dataset": true,
|
||||
"debug": true,
|
||||
"decimal": true,
|
||||
"declare": true,
|
||||
"decode": true,
|
||||
"decoding": true,
|
||||
"decorator": true,
|
||||
"default": true,
|
||||
"define": true,
|
||||
"definition": true,
|
||||
"delegate": true,
|
||||
"delete": true,
|
||||
"deliver": true,
|
||||
"delta": true,
|
||||
"demo": true,
|
||||
"dependency": true,
|
||||
"deploy": true,
|
||||
"describe": true,
|
||||
"description": true,
|
||||
"design": true,
|
||||
"desktop": true,
|
||||
"destroy": true,
|
||||
"detail": true,
|
||||
"detect": true,
|
||||
"develop": true,
|
||||
"developer": true,
|
||||
"device": true,
|
||||
"dialog": true,
|
||||
"dictionary": true,
|
||||
"digest": true,
|
||||
"dimension": true,
|
||||
"direct": true,
|
||||
"directory": true,
|
||||
"disable": true,
|
||||
"discover": true,
|
||||
"dispatch": true,
|
||||
"display": true,
|
||||
"distance": true,
|
||||
"docker": true,
|
||||
"document": true,
|
||||
"domain": true,
|
||||
"download": true,
|
||||
"draft": true,
|
||||
"driver": true,
|
||||
"duration": true,
|
||||
"dynamic": true,
|
||||
"editor": true,
|
||||
"element": true,
|
||||
"email": true,
|
||||
"embedded": true,
|
||||
"empty": true,
|
||||
"enable": true,
|
||||
"encode": true,
|
||||
"encoding": true,
|
||||
"encrypted": true,
|
||||
"endpoint": true,
|
||||
"engine": true,
|
||||
"enterprise": true,
|
||||
"entity": true,
|
||||
"entry": true,
|
||||
"environment": true,
|
||||
"error": true,
|
||||
"escape": true,
|
||||
"event": true,
|
||||
"example": true,
|
||||
"exception": true,
|
||||
"exchange": true,
|
||||
"exclude": true,
|
||||
"execute": true,
|
||||
"exist": true,
|
||||
"expand": true,
|
||||
"expect": true,
|
||||
"experiment": true,
|
||||
"expire": true,
|
||||
"explicit": true,
|
||||
"export": true,
|
||||
"express": true,
|
||||
"extend": true,
|
||||
"extension": true,
|
||||
"external": true,
|
||||
"extract": true,
|
||||
"factory": true,
|
||||
"fallback": true,
|
||||
"feature": true,
|
||||
"fetch": true,
|
||||
"field": true,
|
||||
"file": true,
|
||||
"filename": true,
|
||||
"filter": true,
|
||||
"final": true,
|
||||
"firebase": true,
|
||||
"fixture": true,
|
||||
"flag": true,
|
||||
"float": true,
|
||||
"flush": true,
|
||||
"folder": true,
|
||||
"follow": true,
|
||||
"footer": true,
|
||||
"force": true,
|
||||
"foreign": true,
|
||||
"format": true,
|
||||
"forward": true,
|
||||
"fragment": true,
|
||||
"frame": true,
|
||||
"framework": true,
|
||||
"frequency": true,
|
||||
"frontend": true,
|
||||
"function": true,
|
||||
"gateway": true,
|
||||
"general": true,
|
||||
"generate": true,
|
||||
"generic": true,
|
||||
"github": true,
|
||||
"global": true,
|
||||
"google": true,
|
||||
"gradient": true,
|
||||
"graph": true,
|
||||
"group": true,
|
||||
"guard": true,
|
||||
"guide": true,
|
||||
"handle": true,
|
||||
"handler": true,
|
||||
"hash": true,
|
||||
"hashing": true,
|
||||
"header": true,
|
||||
"health": true,
|
||||
"helper": true,
|
||||
"hidden": true,
|
||||
"history": true,
|
||||
"hook": true,
|
||||
"host": true,
|
||||
"hover": true,
|
||||
"http": true,
|
||||
"https": true,
|
||||
"icon": true,
|
||||
"identifier": true,
|
||||
"ignore": true,
|
||||
"image": true,
|
||||
"implement": true,
|
||||
"import": true,
|
||||
"include": true,
|
||||
"increment": true,
|
||||
"index": true,
|
||||
"indicator": true,
|
||||
"info": true,
|
||||
"infrastructure": true,
|
||||
"inherit": true,
|
||||
"initial": true,
|
||||
"initialize": true,
|
||||
"inject": true,
|
||||
"inline": true,
|
||||
"inner": true,
|
||||
"input": true,
|
||||
"insert": true,
|
||||
"install": true,
|
||||
"instance": true,
|
||||
"integer": true,
|
||||
"integrate": true,
|
||||
"integration": true,
|
||||
"interface": true,
|
||||
"internal": true,
|
||||
"interval": true,
|
||||
"invalid": true,
|
||||
"invoke": true,
|
||||
"issue": true,
|
||||
"item": true,
|
||||
"iterate": true,
|
||||
"iterator": true,
|
||||
"javascript": true,
|
||||
"journal": true,
|
||||
"kernel": true,
|
||||
"label": true,
|
||||
"lambda": true,
|
||||
"language": true,
|
||||
"launch": true,
|
||||
"layer": true,
|
||||
"layout": true,
|
||||
"leader": true,
|
||||
"length": true,
|
||||
"level": true,
|
||||
"library": true,
|
||||
"license": true,
|
||||
"lifecycle": true,
|
||||
"limit": true,
|
||||
"line": true,
|
||||
"link": true,
|
||||
"linux": true,
|
||||
"list": true,
|
||||
"listen": true,
|
||||
"literal": true,
|
||||
"load": true,
|
||||
"loader": true,
|
||||
"local": true,
|
||||
"locale": true,
|
||||
"location": true,
|
||||
"lock": true,
|
||||
"logger": true,
|
||||
"logic": true,
|
||||
"login": true,
|
||||
"lookup": true,
|
||||
"loop": true,
|
||||
"machine": true,
|
||||
"macro": true,
|
||||
"main": true,
|
||||
"manage": true,
|
||||
"manager": true,
|
||||
"manifest": true,
|
||||
"manual": true,
|
||||
"mapper": true,
|
||||
"margin": true,
|
||||
"marker": true,
|
||||
"markup": true,
|
||||
"masked": true,
|
||||
"master": true,
|
||||
"match": true,
|
||||
"matrix": true,
|
||||
"maximum": true,
|
||||
"media": true,
|
||||
"member": true,
|
||||
"memory": true,
|
||||
"menu": true,
|
||||
"merge": true,
|
||||
"message": true,
|
||||
"metadata": true,
|
||||
"method": true,
|
||||
"metric": true,
|
||||
"micro": true,
|
||||
"middleware": true,
|
||||
"migrate": true,
|
||||
"minimum": true,
|
||||
"minor": true,
|
||||
"mobile": true,
|
||||
"mock": true,
|
||||
"model": true,
|
||||
"modify": true,
|
||||
"module": true,
|
||||
"monitor": true,
|
||||
"mount": true,
|
||||
"multi": true,
|
||||
"mutex": true,
|
||||
"name": true,
|
||||
"namespace": true,
|
||||
"native": true,
|
||||
"navigate": true,
|
||||
"network": true,
|
||||
"node": true,
|
||||
"normal": true,
|
||||
"nothing": true,
|
||||
"notice": true,
|
||||
"notification": true,
|
||||
"null": true,
|
||||
"number": true,
|
||||
"object": true,
|
||||
"observe": true,
|
||||
"offset": true,
|
||||
"open": true,
|
||||
"operate": true,
|
||||
"operation": true,
|
||||
"operator": true,
|
||||
"option": true,
|
||||
"oracle": true,
|
||||
"order": true,
|
||||
"origin": true,
|
||||
"original": true,
|
||||
"output": true,
|
||||
"overflow": true,
|
||||
"overlay": true,
|
||||
"override": true,
|
||||
"owner": true,
|
||||
"package": true,
|
||||
"packet": true,
|
||||
"padding": true,
|
||||
"page": true,
|
||||
"panel": true,
|
||||
"parallel": true,
|
||||
"parameter": true,
|
||||
"parent": true,
|
||||
"parse": true,
|
||||
"parser": true,
|
||||
"partial": true,
|
||||
"partition": true,
|
||||
"password_hash": true,
|
||||
"password_reset": true,
|
||||
"patch": true,
|
||||
"path": true,
|
||||
"pattern": true,
|
||||
"payload": true,
|
||||
"pending": true,
|
||||
"percent": true,
|
||||
"perform": true,
|
||||
"period": true,
|
||||
"permission": true,
|
||||
"persist": true,
|
||||
"pipeline": true,
|
||||
"pixel": true,
|
||||
"placeholder": true,
|
||||
"platform": true,
|
||||
"player": true,
|
||||
"plugin": true,
|
||||
"point": true,
|
||||
"pointer": true,
|
||||
"policy": true,
|
||||
"pool": true,
|
||||
"port": true,
|
||||
"portal": true,
|
||||
"position": true,
|
||||
"postgres": true,
|
||||
"prefix": true,
|
||||
"prepare": true,
|
||||
"present": true,
|
||||
"preview": true,
|
||||
"previous": true,
|
||||
"primary": true,
|
||||
"print": true,
|
||||
"priority": true,
|
||||
"private": true,
|
||||
"process": true,
|
||||
"producer": true,
|
||||
"product": true,
|
||||
"profile": true,
|
||||
"program": true,
|
||||
"progress": true,
|
||||
"project": true,
|
||||
"promise": true,
|
||||
"promote": true,
|
||||
"prompt": true,
|
||||
"property": true,
|
||||
"protect": true,
|
||||
"protocol": true,
|
||||
"provide": true,
|
||||
"provider": true,
|
||||
"proxy": true,
|
||||
"public": true,
|
||||
"publish": true,
|
||||
"pull": true,
|
||||
"push": true,
|
||||
"python": true,
|
||||
"query": true,
|
||||
"queue": true,
|
||||
"random": true,
|
||||
"range": true,
|
||||
"rate": true,
|
||||
"react": true,
|
||||
"read": true,
|
||||
"reader": true,
|
||||
"receive": true,
|
||||
"record": true,
|
||||
"recover": true,
|
||||
"redirect": true,
|
||||
"reduce": true,
|
||||
"reference": true,
|
||||
"reflect": true,
|
||||
"refresh": true,
|
||||
"region": true,
|
||||
"register": true,
|
||||
"registry": true,
|
||||
"reject": true,
|
||||
"relate": true,
|
||||
"release": true,
|
||||
"reload": true,
|
||||
"remote": true,
|
||||
"remove": true,
|
||||
"render": true,
|
||||
"replace": true,
|
||||
"replica": true,
|
||||
"report": true,
|
||||
"repository": true,
|
||||
"request": true,
|
||||
"require": true,
|
||||
"reserve": true,
|
||||
"reset": true,
|
||||
"resize": true,
|
||||
"resolve": true,
|
||||
"resource": true,
|
||||
"response": true,
|
||||
"restore": true,
|
||||
"restrict": true,
|
||||
"result": true,
|
||||
"retry": true,
|
||||
"return": true,
|
||||
"reverse": true,
|
||||
"review": true,
|
||||
"revision": true,
|
||||
"role": true,
|
||||
"rollback": true,
|
||||
"root": true,
|
||||
"rotate": true,
|
||||
"route": true,
|
||||
"router": true,
|
||||
"routine": true,
|
||||
"rule": true,
|
||||
"runtime": true,
|
||||
"sample": true,
|
||||
"sandbox": true,
|
||||
"scale": true,
|
||||
"scan": true,
|
||||
"schedule": true,
|
||||
"schema": true,
|
||||
"scope": true,
|
||||
"screen": true,
|
||||
"script": true,
|
||||
"scroll": true,
|
||||
"search": true,
|
||||
"section": true,
|
||||
"secure": true,
|
||||
"security": true,
|
||||
"seed": true,
|
||||
"segment": true,
|
||||
"select": true,
|
||||
"selector": true,
|
||||
"send": true,
|
||||
"sender": true,
|
||||
"sequence": true,
|
||||
"serial": true,
|
||||
"serialize": true,
|
||||
"server": true,
|
||||
"service": true,
|
||||
"session": true,
|
||||
"setting": true,
|
||||
"setup": true,
|
||||
"shadow": true,
|
||||
"shared": true,
|
||||
"shell": true,
|
||||
"shift": true,
|
||||
"short": true,
|
||||
"shutdown": true,
|
||||
"sidebar": true,
|
||||
"signal": true,
|
||||
"signature": true,
|
||||
"signing": true,
|
||||
"simple": true,
|
||||
"simulate": true,
|
||||
"single": true,
|
||||
"size": true,
|
||||
"skeleton": true,
|
||||
"skip": true,
|
||||
"slice": true,
|
||||
"slider": true,
|
||||
"snapshot": true,
|
||||
"socket": true,
|
||||
"software": true,
|
||||
"solution": true,
|
||||
"sort": true,
|
||||
"source": true,
|
||||
"space": true,
|
||||
"span": true,
|
||||
"spec": true,
|
||||
"special": true,
|
||||
"split": true,
|
||||
"spring": true,
|
||||
"stack": true,
|
||||
"stage": true,
|
||||
"standard": true,
|
||||
"start": true,
|
||||
"state": true,
|
||||
"static": true,
|
||||
"status": true,
|
||||
"step": true,
|
||||
"stop": true,
|
||||
"storage": true,
|
||||
"store": true,
|
||||
"strategy": true,
|
||||
"stream": true,
|
||||
"strict": true,
|
||||
"string": true,
|
||||
"struct": true,
|
||||
"structure": true,
|
||||
"style": true,
|
||||
"subject": true,
|
||||
"submit": true,
|
||||
"subscribe": true,
|
||||
"suffix": true,
|
||||
"suggest": true,
|
||||
"suite": true,
|
||||
"summary": true,
|
||||
"super": true,
|
||||
"supply": true,
|
||||
"support": true,
|
||||
"surface": true,
|
||||
"suspend": true,
|
||||
"switch": true,
|
||||
"symbol": true,
|
||||
"sync": true,
|
||||
"syntax": true,
|
||||
"system": true,
|
||||
"table": true,
|
||||
"tablet": true,
|
||||
"target": true,
|
||||
"task": true,
|
||||
"team": true,
|
||||
"template": true,
|
||||
"temporary": true,
|
||||
"tenant": true,
|
||||
"terminal": true,
|
||||
"test": true,
|
||||
"text": true,
|
||||
"theme": true,
|
||||
"thread": true,
|
||||
"threshold": true,
|
||||
"throw": true,
|
||||
"ticket": true,
|
||||
"timeout": true,
|
||||
"timestamp": true,
|
||||
"title": true,
|
||||
"toggle": true,
|
||||
"toolbar": true,
|
||||
"tooltip": true,
|
||||
"topic": true,
|
||||
"total": true,
|
||||
"trace": true,
|
||||
"track": true,
|
||||
"traffic": true,
|
||||
"transaction": true,
|
||||
"transfer": true,
|
||||
"transform": true,
|
||||
"transition": true,
|
||||
"translate": true,
|
||||
"transport": true,
|
||||
"tree": true,
|
||||
"trigger": true,
|
||||
"truncate": true,
|
||||
"tunnel": true,
|
||||
"type": true,
|
||||
"undefined": true,
|
||||
"union": true,
|
||||
"unique": true,
|
||||
"unit": true,
|
||||
"universal": true,
|
||||
"unknown": true,
|
||||
"unpack": true,
|
||||
"update": true,
|
||||
"upgrade": true,
|
||||
"upload": true,
|
||||
"upstream": true,
|
||||
"user": true,
|
||||
"utility": true,
|
||||
"valid": true,
|
||||
"validate": true,
|
||||
"value": true,
|
||||
"variable": true,
|
||||
"variant": true,
|
||||
"vendor": true,
|
||||
"verbose": true,
|
||||
"verify": true,
|
||||
"version": true,
|
||||
"vertex": true,
|
||||
"vertical": true,
|
||||
"video": true,
|
||||
"view": true,
|
||||
"viewport": true,
|
||||
"virtual": true,
|
||||
"visible": true,
|
||||
"visit": true,
|
||||
"volume": true,
|
||||
"watch": true,
|
||||
"webhook": true,
|
||||
"weight": true,
|
||||
"widget": true,
|
||||
"width": true,
|
||||
"window": true,
|
||||
"wizard": true,
|
||||
"worker": true,
|
||||
"workflow": true,
|
||||
"workspace": true,
|
||||
"wrapper": true,
|
||||
"write": true,
|
||||
"writer": true,
|
||||
"zone": true,
|
||||
}
|
||||
|
|
@ -0,0 +1,352 @@
|
|||
// ©AngelaMos | 2026
|
||||
// filter_test.go
|
||||
|
||||
package engine
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/CarterPerez-dev/portia/internal/rules"
|
||||
"github.com/CarterPerez-dev/portia/pkg/types"
|
||||
)
|
||||
|
||||
func TestIsStopword(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := map[string]struct {
|
||||
secret string
|
||||
extra []string
|
||||
want bool
|
||||
}{
|
||||
"exact stopword": {
|
||||
secret: "cache",
|
||||
want: true,
|
||||
},
|
||||
"contains stopword substring": { //nolint:gosec
|
||||
secret: "admin_cache_key_value",
|
||||
want: true,
|
||||
},
|
||||
"random high entropy secret": { //nolint:gosec
|
||||
secret: "xK9mP2vL5nQ8jR3t",
|
||||
want: false,
|
||||
},
|
||||
"extra stopwords match": {
|
||||
secret: "myCustomWord123",
|
||||
extra: []string{"custom"},
|
||||
want: true,
|
||||
},
|
||||
"short stopword ignored": {
|
||||
secret: "abcXYZ",
|
||||
extra: []string{"ab"},
|
||||
want: false,
|
||||
},
|
||||
"real api key not stopped": { //nolint:gosec
|
||||
secret: "AKIAIOSFODNN7EXAMPLE",
|
||||
want: false,
|
||||
},
|
||||
}
|
||||
|
||||
for name, tc := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
got := IsStopword(tc.secret, tc.extra)
|
||||
assert.Equal(t, tc.want, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsPlaceholder(t *testing.T) { //nolint:dupl
|
||||
t.Parallel()
|
||||
|
||||
tests := map[string]struct {
|
||||
secret string
|
||||
want bool
|
||||
}{
|
||||
"example key": {
|
||||
secret: "EXAMPLE_KEY",
|
||||
want: true,
|
||||
},
|
||||
"your-api-key": {
|
||||
secret: "your-api-key",
|
||||
want: true,
|
||||
},
|
||||
"placeholder": {
|
||||
secret: "placeholder",
|
||||
want: true,
|
||||
},
|
||||
"xxxx pattern": {
|
||||
secret: "xxxxxxxxxxxx",
|
||||
want: true,
|
||||
},
|
||||
"stars pattern": {
|
||||
secret: "****",
|
||||
want: true,
|
||||
},
|
||||
"template var dollar": { //nolint:gosec
|
||||
secret: "${SECRET_KEY}",
|
||||
want: true,
|
||||
},
|
||||
"template var braces": { //nolint:gosec
|
||||
secret: "{{api_token}}",
|
||||
want: true,
|
||||
},
|
||||
"CHANGEME": {
|
||||
secret: "CHANGEME",
|
||||
want: true,
|
||||
},
|
||||
"none": {
|
||||
secret: "none",
|
||||
want: true,
|
||||
},
|
||||
"real secret not placeholder": { //nolint:gosec
|
||||
secret: "sk_live_4eC39HqLyjWDarjtT1",
|
||||
want: false,
|
||||
},
|
||||
"real aws key not placeholder": { //nolint:gosec
|
||||
secret: "AKIAIOSFODNN7ABCDEFG",
|
||||
want: false,
|
||||
},
|
||||
}
|
||||
|
||||
for name, tc := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
got := IsPlaceholder(tc.secret)
|
||||
assert.Equal(t, tc.want, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsTemplated(t *testing.T) { //nolint:dupl
|
||||
t.Parallel()
|
||||
|
||||
tests := map[string]struct {
|
||||
secret string
|
||||
want bool
|
||||
}{
|
||||
"dollar brace": {
|
||||
secret: "${DB_PASSWORD}",
|
||||
want: true,
|
||||
},
|
||||
"double brace": { //nolint:gosec
|
||||
secret: "{{api_key}}",
|
||||
want: true,
|
||||
},
|
||||
"os getenv": { //nolint:gosec
|
||||
secret: `os.Getenv("SECRET_KEY")`,
|
||||
want: true,
|
||||
},
|
||||
"process env": { //nolint:gosec
|
||||
secret: "process.env.API_KEY",
|
||||
want: true,
|
||||
},
|
||||
"system getenv": { //nolint:gosec
|
||||
secret: `System.getenv("TOKEN")`,
|
||||
want: true,
|
||||
},
|
||||
"ruby env": { //nolint:gosec
|
||||
secret: `ENV["SECRET"]`,
|
||||
want: true,
|
||||
},
|
||||
"real secret": {
|
||||
secret: "ghp_ABCDEFabcdef1234567890GHIJKL",
|
||||
want: false,
|
||||
},
|
||||
"viper get": {
|
||||
secret: `viper.GetString("api_key")`,
|
||||
want: true,
|
||||
},
|
||||
"python config get": { //nolint:gosec
|
||||
secret: `config.get("secret")`,
|
||||
want: true,
|
||||
},
|
||||
"spring value annotation": { //nolint:gosec
|
||||
secret: `@Value("${db.password}")`,
|
||||
want: true,
|
||||
},
|
||||
"helm template": {
|
||||
secret: `{{ .Values.secretKey }}`,
|
||||
want: true,
|
||||
},
|
||||
}
|
||||
|
||||
for name, tc := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
got := IsTemplated(tc.secret)
|
||||
assert.Equal(t, tc.want, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestHasAssignmentOperator(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := map[string]struct {
|
||||
line string
|
||||
want bool
|
||||
}{
|
||||
"equals sign": {
|
||||
line: `password = "secret123"`,
|
||||
want: true,
|
||||
},
|
||||
"colon": {
|
||||
line: `token: sk_live_abc123def456`,
|
||||
want: true,
|
||||
},
|
||||
"walrus operator": {
|
||||
line: `key := "some_value"`,
|
||||
want: true,
|
||||
},
|
||||
"arrow function": {
|
||||
line: `secret => "value"`,
|
||||
want: true,
|
||||
},
|
||||
"function call no operator": {
|
||||
line: `getPassword(name)`,
|
||||
want: false,
|
||||
},
|
||||
"import statement": {
|
||||
line: `import password_module`,
|
||||
want: false,
|
||||
},
|
||||
"just a variable name": {
|
||||
line: `password_variable`,
|
||||
want: false,
|
||||
},
|
||||
}
|
||||
|
||||
for name, tc := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
got := HasAssignmentOperator(tc.line)
|
||||
assert.Equal(t, tc.want, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsAllowedPath(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := map[string]struct {
|
||||
path string
|
||||
want bool
|
||||
}{
|
||||
"go.sum": {
|
||||
path: "go.sum",
|
||||
want: true,
|
||||
},
|
||||
"package-lock": {
|
||||
path: "package-lock.json",
|
||||
want: true,
|
||||
},
|
||||
"node_modules": {
|
||||
path: "node_modules/pkg/index.js",
|
||||
want: true,
|
||||
},
|
||||
"vendor": {
|
||||
path: "vendor/lib/code.go",
|
||||
want: true,
|
||||
},
|
||||
"git objects": {
|
||||
path: ".git/objects/pack/abc",
|
||||
want: true,
|
||||
},
|
||||
"minified js": {
|
||||
path: "dist/bundle.min.js",
|
||||
want: true,
|
||||
},
|
||||
"png image": {
|
||||
path: "assets/logo.png",
|
||||
want: true,
|
||||
},
|
||||
"source code not allowed": {
|
||||
path: "src/main.go",
|
||||
want: false,
|
||||
},
|
||||
"config file not allowed": {
|
||||
path: "config.yaml",
|
||||
want: false,
|
||||
},
|
||||
"env file not allowed": {
|
||||
path: ".env",
|
||||
want: false,
|
||||
},
|
||||
}
|
||||
|
||||
for name, tc := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
got := IsAllowedPath(
|
||||
tc.path,
|
||||
rules.GlobalPathAllowlist,
|
||||
)
|
||||
assert.Equal(t, tc.want, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilterFinding(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
emptyRule := &types.Rule{
|
||||
Allowlist: types.Allowlist{},
|
||||
}
|
||||
|
||||
tests := map[string]struct {
|
||||
finding *types.Finding
|
||||
rule *types.Rule
|
||||
want bool
|
||||
}{
|
||||
"real secret passes filter": {
|
||||
finding: &types.Finding{ //nolint:gosec
|
||||
Secret: "AKIAIOSFODNN7ABCDEFG",
|
||||
FilePath: "src/config.py",
|
||||
},
|
||||
rule: emptyRule,
|
||||
want: true,
|
||||
},
|
||||
"placeholder filtered out": {
|
||||
finding: &types.Finding{
|
||||
Secret: "your-api-key-here",
|
||||
FilePath: "src/config.py",
|
||||
},
|
||||
rule: emptyRule,
|
||||
want: false,
|
||||
},
|
||||
"template filtered out": {
|
||||
finding: &types.Finding{ //nolint:gosec
|
||||
Secret: "${SECRET_KEY}",
|
||||
FilePath: "src/config.py",
|
||||
},
|
||||
rule: emptyRule,
|
||||
want: false,
|
||||
},
|
||||
"allowed path filtered out": {
|
||||
finding: &types.Finding{
|
||||
Secret: "real_secret_value_here",
|
||||
FilePath: "go.sum",
|
||||
},
|
||||
rule: emptyRule,
|
||||
want: false,
|
||||
},
|
||||
"stopword filtered out": {
|
||||
finding: &types.Finding{
|
||||
Secret: "admin_endpoint_builder",
|
||||
FilePath: "src/config.py",
|
||||
},
|
||||
rule: emptyRule,
|
||||
want: false,
|
||||
},
|
||||
}
|
||||
|
||||
for name, tc := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
got := FilterFinding(tc.finding, tc.rule)
|
||||
assert.Equal(t, tc.want, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,374 @@
|
|||
// ©AngelaMos | 2026
|
||||
// integration_test.go
|
||||
|
||||
package engine_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"sort"
|
||||
"testing"
|
||||
|
||||
"github.com/CarterPerez-dev/portia/internal/engine"
|
||||
"github.com/CarterPerez-dev/portia/internal/rules"
|
||||
"github.com/CarterPerez-dev/portia/internal/source"
|
||||
"github.com/CarterPerez-dev/portia/pkg/types"
|
||||
)
|
||||
|
||||
func testdataDir(t *testing.T) string {
|
||||
t.Helper()
|
||||
_, filename, _, ok := runtime.Caller(0)
|
||||
if !ok {
|
||||
t.Fatal("unable to determine test file location")
|
||||
}
|
||||
return filepath.Join(
|
||||
filepath.Dir(filename), "..", "..", "testdata", "fixtures",
|
||||
)
|
||||
}
|
||||
|
||||
func setupPipeline(t *testing.T) (*engine.Pipeline, *rules.Registry) {
|
||||
t.Helper()
|
||||
reg := rules.NewRegistry()
|
||||
rules.RegisterBuiltins(reg)
|
||||
return engine.NewPipeline(reg), reg
|
||||
}
|
||||
|
||||
func findByRuleID(
|
||||
findings []types.Finding, ruleID string,
|
||||
) *types.Finding {
|
||||
for i := range findings {
|
||||
if findings[i].RuleID == ruleID {
|
||||
return &findings[i]
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestIntegrationFullScan(t *testing.T) {
|
||||
dir := testdataDir(t)
|
||||
if _, err := os.Stat(dir); os.IsNotExist(err) {
|
||||
t.Skip("testdata/fixtures not found")
|
||||
}
|
||||
|
||||
p, _ := setupPipeline(t)
|
||||
src := source.NewDirectory(dir, 0, nil)
|
||||
|
||||
result, err := p.Run(context.Background(), src)
|
||||
if err != nil {
|
||||
t.Fatalf("pipeline run failed: %v", err)
|
||||
}
|
||||
|
||||
if len(result.Findings) == 0 {
|
||||
t.Fatal("expected findings, got none")
|
||||
}
|
||||
|
||||
ruleIDs := make(map[string]bool)
|
||||
for _, f := range result.Findings {
|
||||
ruleIDs[f.RuleID] = true
|
||||
}
|
||||
|
||||
expectedRules := []string{
|
||||
"aws-access-key-id",
|
||||
"stripe-live-secret",
|
||||
"ssh-private-key-rsa",
|
||||
"generic-password",
|
||||
}
|
||||
|
||||
for _, id := range expectedRules {
|
||||
if !ruleIDs[id] {
|
||||
t.Errorf("expected finding for rule %q, not found", id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntegrationAWSKeyDetection(t *testing.T) {
|
||||
dir := testdataDir(t)
|
||||
p, _ := setupPipeline(t)
|
||||
src := source.NewDirectory(dir, 0, nil)
|
||||
|
||||
result, err := p.Run(context.Background(), src)
|
||||
if err != nil {
|
||||
t.Fatalf("pipeline run failed: %v", err)
|
||||
}
|
||||
|
||||
f := findByRuleID(result.Findings, "aws-access-key-id")
|
||||
if f == nil {
|
||||
t.Fatal("aws-access-key-id finding not found")
|
||||
}
|
||||
|
||||
if f.Severity != types.SeverityCritical {
|
||||
t.Errorf("expected CRITICAL, got %s", f.Severity)
|
||||
}
|
||||
|
||||
if f.FilePath != "config.py" {
|
||||
t.Errorf("expected config.py, got %s", f.FilePath)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntegrationStripeKeyDetection(t *testing.T) {
|
||||
dir := testdataDir(t)
|
||||
p, _ := setupPipeline(t)
|
||||
src := source.NewDirectory(dir, 0, nil)
|
||||
|
||||
result, err := p.Run(context.Background(), src)
|
||||
if err != nil {
|
||||
t.Fatalf("pipeline run failed: %v", err)
|
||||
}
|
||||
|
||||
f := findByRuleID(result.Findings, "stripe-live-secret")
|
||||
if f == nil {
|
||||
t.Fatal("stripe-live-secret finding not found")
|
||||
}
|
||||
|
||||
if f.Severity != types.SeverityCritical {
|
||||
t.Errorf("expected CRITICAL, got %s", f.Severity)
|
||||
}
|
||||
|
||||
if f.FilePath != "payment.env" {
|
||||
t.Errorf("expected payment.env, got %s", f.FilePath)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntegrationPrivateKeyDetection(t *testing.T) {
|
||||
dir := testdataDir(t)
|
||||
p, _ := setupPipeline(t)
|
||||
src := source.NewDirectory(dir, 0, nil)
|
||||
|
||||
result, err := p.Run(context.Background(), src)
|
||||
if err != nil {
|
||||
t.Fatalf("pipeline run failed: %v", err)
|
||||
}
|
||||
|
||||
f := findByRuleID(result.Findings, "ssh-private-key-rsa")
|
||||
if f == nil {
|
||||
t.Fatal("ssh-private-key-rsa finding not found")
|
||||
}
|
||||
|
||||
if f.Severity != types.SeverityCritical {
|
||||
t.Errorf("expected CRITICAL, got %s", f.Severity)
|
||||
}
|
||||
|
||||
if f.FilePath != "key.pem" {
|
||||
t.Errorf("expected key.pem, got %s", f.FilePath)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntegrationPasswordDetection(t *testing.T) {
|
||||
dir := testdataDir(t)
|
||||
p, _ := setupPipeline(t)
|
||||
src := source.NewDirectory(dir, 0, nil)
|
||||
|
||||
result, err := p.Run(context.Background(), src)
|
||||
if err != nil {
|
||||
t.Fatalf("pipeline run failed: %v", err)
|
||||
}
|
||||
|
||||
f := findByRuleID(result.Findings, "generic-password")
|
||||
if f == nil {
|
||||
t.Fatal("generic-password finding not found")
|
||||
}
|
||||
|
||||
if f.Severity != types.SeverityHigh {
|
||||
t.Errorf("expected HIGH, got %s", f.Severity)
|
||||
}
|
||||
|
||||
if f.Entropy <= 0 {
|
||||
t.Error("expected positive entropy for password")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntegrationGitHubTokenDetection(t *testing.T) {
|
||||
dir := testdataDir(t)
|
||||
p, _ := setupPipeline(t)
|
||||
src := source.NewDirectory(dir, 0, nil)
|
||||
|
||||
result, err := p.Run(context.Background(), src)
|
||||
if err != nil {
|
||||
t.Fatalf("pipeline run failed: %v", err)
|
||||
}
|
||||
|
||||
f := findByRuleID(result.Findings, "github-pat-classic")
|
||||
if f == nil {
|
||||
f = findByRuleID(result.Findings, "github-pat-fine")
|
||||
if f == nil {
|
||||
t.Fatal(
|
||||
"github-pat-classic or github-pat-fine finding not found",
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
if f.Severity != types.SeverityCritical {
|
||||
t.Errorf("expected CRITICAL, got %s", f.Severity)
|
||||
}
|
||||
|
||||
if f.FilePath != "github_token.js" {
|
||||
t.Errorf("expected github_token.js, got %s", f.FilePath)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntegrationConnectionStringDetection(t *testing.T) {
|
||||
dir := testdataDir(t)
|
||||
p, _ := setupPipeline(t)
|
||||
src := source.NewDirectory(dir, 0, nil)
|
||||
|
||||
result, err := p.Run(context.Background(), src)
|
||||
if err != nil {
|
||||
t.Fatalf("pipeline run failed: %v", err)
|
||||
}
|
||||
|
||||
f := findByRuleID(result.Findings, "postgres-connection")
|
||||
if f == nil {
|
||||
t.Fatal("postgres-connection finding not found")
|
||||
}
|
||||
|
||||
if f.FilePath != "connection.yml" {
|
||||
t.Errorf("expected connection.yml, got %s", f.FilePath)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntegrationNoFalsePositivesOnSafeFiles(t *testing.T) {
|
||||
dir := testdataDir(t)
|
||||
p, _ := setupPipeline(t)
|
||||
src := source.NewDirectory(dir, 0, nil)
|
||||
|
||||
result, err := p.Run(context.Background(), src)
|
||||
if err != nil {
|
||||
t.Fatalf("pipeline run failed: %v", err)
|
||||
}
|
||||
|
||||
for _, f := range result.Findings {
|
||||
if f.FilePath == "safe.txt" {
|
||||
t.Errorf(
|
||||
"unexpected finding in safe.txt: %s (%s)",
|
||||
f.RuleID, f.Secret,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntegrationTemplatesFiltered(t *testing.T) {
|
||||
dir := testdataDir(t)
|
||||
p, _ := setupPipeline(t)
|
||||
src := source.NewDirectory(dir, 0, nil)
|
||||
|
||||
result, err := p.Run(context.Background(), src)
|
||||
if err != nil {
|
||||
t.Fatalf("pipeline run failed: %v", err)
|
||||
}
|
||||
|
||||
for _, f := range result.Findings {
|
||||
if f.FilePath == "template.env" {
|
||||
t.Errorf(
|
||||
"unexpected finding in template.env: %s (%s)",
|
||||
f.RuleID, f.Secret,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntegrationExcludePaths(t *testing.T) {
|
||||
dir := testdataDir(t)
|
||||
p, _ := setupPipeline(t)
|
||||
src := source.NewDirectory(dir, 0, []string{"*.py"})
|
||||
|
||||
result, err := p.Run(context.Background(), src)
|
||||
if err != nil {
|
||||
t.Fatalf("pipeline run failed: %v", err)
|
||||
}
|
||||
|
||||
for _, f := range result.Findings {
|
||||
if f.FilePath == "config.py" {
|
||||
t.Error("config.py should be excluded")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntegrationDisableRules(t *testing.T) {
|
||||
dir := testdataDir(t)
|
||||
reg := rules.NewRegistry()
|
||||
rules.RegisterBuiltins(reg)
|
||||
reg.Disable("aws-access-key-id", "stripe-live-secret")
|
||||
|
||||
p := engine.NewPipeline(reg)
|
||||
src := source.NewDirectory(dir, 0, nil)
|
||||
|
||||
result, err := p.Run(context.Background(), src)
|
||||
if err != nil {
|
||||
t.Fatalf("pipeline run failed: %v", err)
|
||||
}
|
||||
|
||||
for _, f := range result.Findings {
|
||||
if f.RuleID == "aws-access-key-id" {
|
||||
t.Error("aws-access-key-id should be disabled")
|
||||
}
|
||||
if f.RuleID == "stripe-live-secret" {
|
||||
t.Error("stripe-live-secret should be disabled")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntegrationFindingsSortBySeverity(t *testing.T) {
|
||||
dir := testdataDir(t)
|
||||
p, _ := setupPipeline(t)
|
||||
src := source.NewDirectory(dir, 0, nil)
|
||||
|
||||
result, err := p.Run(context.Background(), src)
|
||||
if err != nil {
|
||||
t.Fatalf("pipeline run failed: %v", err)
|
||||
}
|
||||
|
||||
sorted := make([]types.Finding, len(result.Findings))
|
||||
copy(sorted, result.Findings)
|
||||
sort.Slice(sorted, func(i, j int) bool {
|
||||
return sorted[i].Severity.Rank() < sorted[j].Severity.Rank()
|
||||
})
|
||||
|
||||
if len(sorted) > 0 &&
|
||||
sorted[0].Severity != types.SeverityCritical {
|
||||
t.Errorf(
|
||||
"expected first sorted finding to be CRITICAL, got %s",
|
||||
sorted[0].Severity,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntegrationResultMetadata(t *testing.T) {
|
||||
dir := testdataDir(t)
|
||||
p, reg := setupPipeline(t)
|
||||
src := source.NewDirectory(dir, 0, nil)
|
||||
|
||||
result, err := p.Run(context.Background(), src)
|
||||
if err != nil {
|
||||
t.Fatalf("pipeline run failed: %v", err)
|
||||
}
|
||||
|
||||
if result.TotalRules != reg.Len() {
|
||||
t.Errorf(
|
||||
"expected %d rules, got %d",
|
||||
reg.Len(), result.TotalRules,
|
||||
)
|
||||
}
|
||||
|
||||
if result.TotalFindings < len(result.Findings) {
|
||||
t.Errorf(
|
||||
"total findings %d < unique findings %d",
|
||||
result.TotalFindings, len(result.Findings),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntegrationContextCancellation(t *testing.T) {
|
||||
dir := testdataDir(t)
|
||||
p, _ := setupPipeline(t)
|
||||
src := source.NewDirectory(dir, 0, nil)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
_, err := p.Run(ctx, src)
|
||||
if err == nil {
|
||||
t.Log("expected error or empty result from cancelled context")
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,147 @@
|
|||
// ©AngelaMos | 2026
|
||||
// pipeline.go
|
||||
|
||||
package engine
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"runtime"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"golang.org/x/sync/errgroup"
|
||||
|
||||
"github.com/CarterPerez-dev/portia/internal/rules"
|
||||
"github.com/CarterPerez-dev/portia/internal/source"
|
||||
"github.com/CarterPerez-dev/portia/internal/ui"
|
||||
"github.com/CarterPerez-dev/portia/pkg/types"
|
||||
)
|
||||
|
||||
type Pipeline struct {
|
||||
registry *rules.Registry
|
||||
detector *Detector
|
||||
workers int
|
||||
verbose bool
|
||||
}
|
||||
|
||||
func NewPipeline(reg *rules.Registry) *Pipeline {
|
||||
workers := runtime.NumCPU()
|
||||
if workers < 2 {
|
||||
workers = 2
|
||||
}
|
||||
if workers > 16 {
|
||||
workers = 16
|
||||
}
|
||||
|
||||
return &Pipeline{
|
||||
registry: reg,
|
||||
detector: NewDetector(reg),
|
||||
workers: workers,
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Pipeline) SetVerbose(v bool) {
|
||||
p.verbose = v
|
||||
}
|
||||
|
||||
func (p *Pipeline) Run( //nolint:gocognit
|
||||
ctx context.Context, src source.Source,
|
||||
) (*types.ScanResult, error) {
|
||||
chunks := make(chan types.Chunk, p.workers*4)
|
||||
findingsCh := make(chan types.Finding, p.workers*4)
|
||||
|
||||
g, gctx := errgroup.WithContext(ctx)
|
||||
|
||||
g.Go(func() error {
|
||||
defer close(chunks)
|
||||
return src.Chunks(gctx, chunks)
|
||||
})
|
||||
|
||||
var detectWg sync.WaitGroup
|
||||
var seenFiles sync.Map
|
||||
var filesScanned atomic.Int64
|
||||
|
||||
for range p.workers {
|
||||
detectWg.Add(1)
|
||||
g.Go(func() error {
|
||||
defer detectWg.Done()
|
||||
for chunk := range chunks {
|
||||
if gctx.Err() != nil {
|
||||
return gctx.Err()
|
||||
}
|
||||
|
||||
if _, loaded := seenFiles.LoadOrStore(
|
||||
chunk.FilePath,
|
||||
true,
|
||||
); !loaded {
|
||||
filesScanned.Add(1)
|
||||
if p.verbose {
|
||||
fmt.Printf(" %s %s\n",
|
||||
ui.Dim("scanning"),
|
||||
ui.Dim(chunk.FilePath))
|
||||
}
|
||||
}
|
||||
|
||||
results := p.detector.Detect(chunk)
|
||||
for _, f := range results {
|
||||
select {
|
||||
case <-gctx.Done():
|
||||
return gctx.Err()
|
||||
case findingsCh <- f:
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
go func() {
|
||||
detectWg.Wait()
|
||||
close(findingsCh)
|
||||
}()
|
||||
|
||||
var mu sync.Mutex
|
||||
var allFindings []types.Finding
|
||||
|
||||
g.Go(func() error {
|
||||
for f := range findingsCh {
|
||||
mu.Lock()
|
||||
allFindings = append(allFindings, f)
|
||||
mu.Unlock()
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
if err := g.Wait(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
totalFiles := int(filesScanned.Load())
|
||||
|
||||
result := &types.ScanResult{
|
||||
Findings: dedup(allFindings),
|
||||
TotalFiles: totalFiles,
|
||||
TotalRules: p.registry.Len(),
|
||||
TotalFindings: len(allFindings),
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func dedup(findings []types.Finding) []types.Finding {
|
||||
seen := make(map[string]bool)
|
||||
var unique []types.Finding
|
||||
|
||||
for _, f := range findings {
|
||||
key := f.RuleID + "|" + f.FilePath + "|" +
|
||||
f.Secret + "|" + f.CommitSHA
|
||||
if seen[key] {
|
||||
continue
|
||||
}
|
||||
seen[key] = true
|
||||
unique = append(unique, f)
|
||||
}
|
||||
|
||||
return unique
|
||||
}
|
||||
|
|
@ -0,0 +1,147 @@
|
|||
// ©AngelaMos | 2026
|
||||
// pipeline_test.go
|
||||
|
||||
package engine
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/CarterPerez-dev/portia/internal/rules"
|
||||
"github.com/CarterPerez-dev/portia/internal/source"
|
||||
)
|
||||
|
||||
func TestPipelineDirectoryScan(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dir := t.TempDir()
|
||||
require.NoError(t, os.WriteFile( //nolint:gosec
|
||||
filepath.Join(dir, "config.py"),
|
||||
[]byte(`api_key = "AKIAIOSFODNN7EXAMPLE"`+"\n"),
|
||||
0o644,
|
||||
))
|
||||
require.NoError(t, os.WriteFile( //nolint:gosec
|
||||
filepath.Join(dir, "clean.go"),
|
||||
[]byte("package main\n\nfunc main() {}\n"),
|
||||
0o644,
|
||||
))
|
||||
|
||||
reg := testRegistry()
|
||||
p := NewPipeline(reg)
|
||||
src := source.NewDirectory(dir, 0, nil)
|
||||
|
||||
result, err := p.Run(context.Background(), src)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, result)
|
||||
assert.GreaterOrEqual(t, len(result.Findings), 1)
|
||||
|
||||
found := false
|
||||
for _, f := range result.Findings {
|
||||
if f.RuleID == "test-aws-key" {
|
||||
found = true
|
||||
assert.Equal(t, "AKIAIOSFODNN7EXAMPLE", f.Secret)
|
||||
}
|
||||
}
|
||||
assert.True(t, found, "expected to find AWS key")
|
||||
}
|
||||
|
||||
func TestPipelineNoFindings(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dir := t.TempDir()
|
||||
require.NoError(t, os.WriteFile( //nolint:gosec
|
||||
filepath.Join(dir, "main.go"),
|
||||
[]byte("package main\n\nfunc main() {}\n"),
|
||||
0o644,
|
||||
))
|
||||
|
||||
reg := testRegistry()
|
||||
p := NewPipeline(reg)
|
||||
src := source.NewDirectory(dir, 0, nil)
|
||||
|
||||
result, err := p.Run(context.Background(), src)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, result.Findings)
|
||||
}
|
||||
|
||||
func TestPipelineMultipleSecrets(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dir := t.TempDir()
|
||||
require.NoError(t, os.WriteFile( //nolint:gosec
|
||||
filepath.Join(dir, "secrets.env"),
|
||||
[]byte(
|
||||
`AWS_KEY=AKIAIOSFODNN7EXAMPLE`+"\n"+
|
||||
`STRIPE_KEY=sk_live_4eC39HqLyjWDarjtT1zdp7dc`+"\n"+
|
||||
`password = "xK9mP2vL5nQ8jR3t"`+"\n",
|
||||
),
|
||||
0o644,
|
||||
))
|
||||
|
||||
reg := testRegistry()
|
||||
p := NewPipeline(reg)
|
||||
src := source.NewDirectory(dir, 0, nil)
|
||||
|
||||
result, err := p.Run(context.Background(), src)
|
||||
require.NoError(t, err)
|
||||
assert.GreaterOrEqual(t, len(result.Findings), 2)
|
||||
}
|
||||
|
||||
func TestPipelineContextCancel(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dir := t.TempDir()
|
||||
for i := range 20 {
|
||||
name := "file" + string(rune('A'+i)) + ".txt"
|
||||
require.NoError(t, os.WriteFile( //nolint:gosec
|
||||
filepath.Join(dir, name),
|
||||
[]byte(`AKIAIOSFODNN7EXAMPLE`+"\n"),
|
||||
0o644,
|
||||
))
|
||||
}
|
||||
|
||||
reg := testRegistry()
|
||||
p := NewPipeline(reg)
|
||||
src := source.NewDirectory(dir, 0, nil)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
_, err := p.Run(ctx, src)
|
||||
assert.Error(t, err)
|
||||
}
|
||||
|
||||
func TestPipelineDedup(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
reg := rules.NewRegistry()
|
||||
rules.RegisterBuiltins(reg)
|
||||
p := NewPipeline(reg)
|
||||
|
||||
dir := t.TempDir()
|
||||
require.NoError(t, os.WriteFile( //nolint:gosec
|
||||
filepath.Join(dir, "dup.txt"),
|
||||
[]byte(
|
||||
`AKIAIOSFODNN7EXAMPLE`+"\n"+
|
||||
`AKIAIOSFODNN7EXAMPLE`+"\n",
|
||||
),
|
||||
0o644,
|
||||
))
|
||||
|
||||
src := source.NewDirectory(dir, 0, nil)
|
||||
result, err := p.Run(context.Background(), src)
|
||||
require.NoError(t, err)
|
||||
|
||||
awsCount := 0
|
||||
for _, f := range result.Findings {
|
||||
if f.Secret == "AKIAIOSFODNN7EXAMPLE" { //nolint:gosec
|
||||
awsCount++
|
||||
}
|
||||
}
|
||||
assert.LessOrEqual(t, awsCount, 2)
|
||||
}
|
||||
|
|
@ -0,0 +1,177 @@
|
|||
// ©AngelaMos | 2026
|
||||
// client.go
|
||||
|
||||
package hibp
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"crypto/sha1"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
lru "github.com/hashicorp/golang-lru/v2"
|
||||
"github.com/sony/gobreaker/v2"
|
||||
"golang.org/x/time/rate"
|
||||
)
|
||||
|
||||
type Result struct {
|
||||
Breached bool
|
||||
Count int
|
||||
}
|
||||
|
||||
type Client struct {
|
||||
httpClient *http.Client
|
||||
baseURL string
|
||||
cache *lru.Cache[string, Result]
|
||||
breaker *gobreaker.CircuitBreaker[Result]
|
||||
limiter *rate.Limiter
|
||||
}
|
||||
|
||||
func NewClient() *Client {
|
||||
cache, err := lru.New[string, Result](10000)
|
||||
if err != nil {
|
||||
panic(fmt.Errorf("create HIBP cache: %w", err))
|
||||
}
|
||||
|
||||
breaker := gobreaker.NewCircuitBreaker[Result](
|
||||
gobreaker.Settings{
|
||||
Name: "hibp",
|
||||
MaxRequests: 3,
|
||||
Interval: 30 * time.Second,
|
||||
Timeout: 60 * time.Second,
|
||||
ReadyToTrip: func(
|
||||
counts gobreaker.Counts,
|
||||
) bool {
|
||||
return counts.ConsecutiveFailures > 5
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
return &Client{
|
||||
httpClient: &http.Client{
|
||||
Timeout: 10 * time.Second,
|
||||
},
|
||||
baseURL: "https://api.pwnedpasswords.com/range/",
|
||||
cache: cache,
|
||||
breaker: breaker,
|
||||
limiter: rate.NewLimiter(rate.Every(200*time.Millisecond), 5),
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Client) Check(
|
||||
ctx context.Context, secret string,
|
||||
) (Result, error) {
|
||||
hash := sha1Hash(secret)
|
||||
prefix := hash[:5]
|
||||
suffix := hash[5:]
|
||||
|
||||
if cached, ok := c.cache.Get(hash); ok {
|
||||
return cached, nil
|
||||
}
|
||||
|
||||
if err := c.limiter.Wait(ctx); err != nil {
|
||||
return Result{}, fmt.Errorf("rate limit: %w", err)
|
||||
}
|
||||
|
||||
result, err := c.breaker.Execute(func() (Result, error) {
|
||||
return c.queryAPI(ctx, prefix, suffix)
|
||||
})
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
|
||||
c.cache.Add(hash, result)
|
||||
return result, nil
|
||||
}
|
||||
|
||||
const maxRetries = 3
|
||||
|
||||
func (c *Client) queryAPI(
|
||||
ctx context.Context, prefix, suffix string,
|
||||
) (Result, error) {
|
||||
url := c.baseURL + prefix
|
||||
|
||||
for attempt := range maxRetries {
|
||||
if attempt > 0 {
|
||||
backoff := time.Duration(attempt) * 2 * time.Second
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return Result{}, ctx.Err()
|
||||
case <-time.After(backoff):
|
||||
}
|
||||
}
|
||||
|
||||
result, retry, err := c.doQuery(ctx, url, suffix)
|
||||
if !retry {
|
||||
return result, err
|
||||
}
|
||||
}
|
||||
|
||||
return Result{}, fmt.Errorf("HIBP API: retries exhausted (429)")
|
||||
}
|
||||
|
||||
func (c *Client) doQuery(
|
||||
ctx context.Context, url, suffix string,
|
||||
) (Result, bool, error) {
|
||||
req, err := http.NewRequestWithContext(
|
||||
ctx, http.MethodGet, url, nil,
|
||||
)
|
||||
if err != nil {
|
||||
return Result{}, false,
|
||||
fmt.Errorf("create request: %w", err)
|
||||
}
|
||||
req.Header.Set("User-Agent", "portia-secrets-scanner/1.0")
|
||||
req.Header.Set("Add-Padding", "true")
|
||||
|
||||
resp, err := c.httpClient.Do(req) //nolint:gosec
|
||||
if err != nil {
|
||||
return Result{}, false,
|
||||
fmt.Errorf("HIBP request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close() //nolint:errcheck
|
||||
|
||||
if resp.StatusCode == http.StatusTooManyRequests {
|
||||
return Result{}, true, nil
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return Result{}, false, fmt.Errorf(
|
||||
"HIBP API status: %d", resp.StatusCode,
|
||||
)
|
||||
}
|
||||
|
||||
scanner := bufio.NewScanner(resp.Body)
|
||||
upperSuffix := strings.ToUpper(suffix)
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
parts := strings.SplitN(line, ":", 2)
|
||||
if len(parts) != 2 {
|
||||
continue
|
||||
}
|
||||
if parts[0] == upperSuffix {
|
||||
trimmed := strings.TrimSpace(parts[1])
|
||||
count, err := strconv.Atoi(trimmed)
|
||||
if err != nil {
|
||||
return Result{}, false, fmt.Errorf(
|
||||
"parse breach count %q: %w",
|
||||
trimmed,
|
||||
err,
|
||||
)
|
||||
}
|
||||
return Result{Breached: true, Count: count},
|
||||
false, nil
|
||||
}
|
||||
}
|
||||
|
||||
return Result{Breached: false, Count: 0}, false, nil
|
||||
}
|
||||
|
||||
func sha1Hash(s string) string {
|
||||
h := sha1.New()
|
||||
h.Write([]byte(s))
|
||||
return fmt.Sprintf("%X", h.Sum(nil))
|
||||
}
|
||||
|
|
@ -0,0 +1,203 @@
|
|||
// ©AngelaMos | 2026
|
||||
// client_test.go
|
||||
|
||||
package hibp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestSha1Hash(t *testing.T) {
|
||||
t.Parallel()
|
||||
got := sha1Hash("password")
|
||||
assert.Equal(t,
|
||||
"5BAA61E4C9B93F3F0682250B6CF8331B7EE68FD8",
|
||||
got,
|
||||
)
|
||||
}
|
||||
|
||||
func TestClientCheckBreached(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
hash := sha1Hash("password")
|
||||
prefix := hash[:5]
|
||||
suffix := hash[5:]
|
||||
|
||||
server := httptest.NewServer(
|
||||
http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
assert.Equal(t, "/range/"+prefix, r.URL.Path)
|
||||
w.WriteHeader(http.StatusOK)
|
||||
fmt.Fprintf( //nolint:errcheck
|
||||
w,
|
||||
"%s:3861493\n",
|
||||
suffix,
|
||||
)
|
||||
fmt.Fprintf( //nolint:errcheck
|
||||
w,
|
||||
"0000000000000000000000000000DEAD0:0\n",
|
||||
)
|
||||
}),
|
||||
)
|
||||
defer server.Close()
|
||||
|
||||
client := NewClient()
|
||||
client.baseURL = server.URL + "/range/"
|
||||
|
||||
result, err := client.Check(context.Background(), "password")
|
||||
require.NoError(t, err)
|
||||
assert.True(t, result.Breached)
|
||||
assert.Equal(t, 3861493, result.Count)
|
||||
}
|
||||
|
||||
func TestClientCheckClean(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
server := httptest.NewServer(
|
||||
http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
fmt.Fprintf(w, //nolint:errcheck
|
||||
"0000000000000000000000000000AAAA0:5\n")
|
||||
fmt.Fprintf( //nolint:errcheck
|
||||
w,
|
||||
"0000000000000000000000000000BBBB0:2\n",
|
||||
)
|
||||
}),
|
||||
)
|
||||
defer server.Close()
|
||||
|
||||
client := NewClient()
|
||||
client.baseURL = server.URL + "/range/"
|
||||
|
||||
result, err := client.Check(
|
||||
context.Background(), "unique_password_not_in_breach",
|
||||
)
|
||||
require.NoError(t, err)
|
||||
assert.False(t, result.Breached)
|
||||
assert.Equal(t, 0, result.Count)
|
||||
}
|
||||
|
||||
func TestClientCachesResults(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
callCount := 0
|
||||
server := httptest.NewServer(
|
||||
http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
callCount++
|
||||
hash := sha1Hash("cached_secret")
|
||||
suffix := hash[5:]
|
||||
w.WriteHeader(http.StatusOK)
|
||||
fmt.Fprintf(w, "%s:42\n", suffix) //nolint:errcheck
|
||||
}),
|
||||
)
|
||||
defer server.Close()
|
||||
|
||||
client := NewClient()
|
||||
client.baseURL = server.URL + "/range/"
|
||||
|
||||
result1, err := client.Check(
|
||||
context.Background(), "cached_secret",
|
||||
)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, result1.Breached)
|
||||
|
||||
result2, err := client.Check(
|
||||
context.Background(), "cached_secret",
|
||||
)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, result2.Breached)
|
||||
assert.Equal(t, 42, result2.Count)
|
||||
assert.Equal(t, 1, callCount)
|
||||
}
|
||||
|
||||
func TestClientHandlesServerError(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
server := httptest.NewServer(
|
||||
http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
}),
|
||||
)
|
||||
defer server.Close()
|
||||
|
||||
client := NewClient()
|
||||
client.baseURL = server.URL + "/range/"
|
||||
|
||||
_, err := client.Check(context.Background(), "test")
|
||||
assert.Error(t, err)
|
||||
}
|
||||
|
||||
func TestClientRetries429(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
hash := sha1Hash("retryable")
|
||||
suffix := hash[5:]
|
||||
|
||||
attempts := 0
|
||||
server := httptest.NewServer(
|
||||
http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
attempts++
|
||||
if attempts < 3 {
|
||||
w.WriteHeader(http.StatusTooManyRequests)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
fmt.Fprintf(w, "%s:10\n", suffix) //nolint:errcheck
|
||||
}),
|
||||
)
|
||||
defer server.Close()
|
||||
|
||||
client := NewClient()
|
||||
client.baseURL = server.URL + "/range/"
|
||||
|
||||
result, err := client.Check(context.Background(), "retryable")
|
||||
require.NoError(t, err)
|
||||
assert.True(t, result.Breached)
|
||||
assert.Equal(t, 10, result.Count)
|
||||
assert.Equal(t, 3, attempts)
|
||||
}
|
||||
|
||||
func TestClientRetries429Exhausted(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
server := httptest.NewServer(
|
||||
http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusTooManyRequests)
|
||||
}),
|
||||
)
|
||||
defer server.Close()
|
||||
|
||||
client := NewClient()
|
||||
client.baseURL = server.URL + "/range/"
|
||||
|
||||
_, err := client.Check(context.Background(), "always429")
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "retries exhausted")
|
||||
}
|
||||
|
||||
func TestClientContextCancellation(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
server := httptest.NewServer(
|
||||
http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
fmt.Fprintf(w, "AAAA0:1\n") //nolint:errcheck
|
||||
}),
|
||||
)
|
||||
defer server.Close()
|
||||
|
||||
client := NewClient()
|
||||
client.baseURL = server.URL + "/range/"
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
_, err := client.Check(ctx, "test")
|
||||
assert.Error(t, err)
|
||||
}
|
||||
|
|
@ -0,0 +1,77 @@
|
|||
// ©AngelaMos | 2026
|
||||
// json.go
|
||||
|
||||
package reporter
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
|
||||
"github.com/CarterPerez-dev/portia/pkg/types"
|
||||
)
|
||||
|
||||
type JSON struct{}
|
||||
|
||||
type jsonOutput struct {
|
||||
Findings []jsonFinding `json:"findings"`
|
||||
Summary jsonSummary `json:"summary"`
|
||||
}
|
||||
|
||||
type jsonFinding struct {
|
||||
RuleID string `json:"rule_id"`
|
||||
Description string `json:"description"`
|
||||
Severity string `json:"severity"`
|
||||
Secret string `json:"secret"` //nolint:gosec
|
||||
Entropy float64 `json:"entropy,omitempty"`
|
||||
File string `json:"file"`
|
||||
Line int `json:"line"`
|
||||
Commit string `json:"commit,omitempty"`
|
||||
Author string `json:"author,omitempty"`
|
||||
HIBPStatus string `json:"hibp_status,omitempty"`
|
||||
BreachCount int `json:"breach_count,omitempty"`
|
||||
}
|
||||
|
||||
type jsonSummary struct {
|
||||
TotalFiles int `json:"total_files"`
|
||||
TotalFindings int `json:"total_findings"`
|
||||
TotalRules int `json:"total_rules"`
|
||||
Duration string `json:"duration"`
|
||||
HIBPChecked int `json:"hibp_checked,omitempty"`
|
||||
HIBPBreached int `json:"hibp_breached,omitempty"`
|
||||
}
|
||||
|
||||
func (j *JSON) Report(
|
||||
w io.Writer, result *types.ScanResult,
|
||||
) error {
|
||||
output := jsonOutput{
|
||||
Summary: jsonSummary{
|
||||
TotalFiles: result.TotalFiles,
|
||||
TotalFindings: len(result.Findings),
|
||||
TotalRules: result.TotalRules,
|
||||
Duration: result.Duration.String(),
|
||||
HIBPChecked: result.HIBPChecked,
|
||||
HIBPBreached: result.HIBPBreached,
|
||||
},
|
||||
}
|
||||
|
||||
output.Findings = make([]jsonFinding, len(result.Findings))
|
||||
for i, f := range result.Findings {
|
||||
output.Findings[i] = jsonFinding{
|
||||
RuleID: f.RuleID,
|
||||
Description: f.Description,
|
||||
Severity: f.Severity.String(),
|
||||
Secret: maskSecret(f.Secret),
|
||||
Entropy: f.Entropy,
|
||||
File: f.FilePath,
|
||||
Line: f.LineNumber,
|
||||
Commit: f.CommitSHA,
|
||||
Author: f.Author,
|
||||
HIBPStatus: f.HIBPStatus.String(),
|
||||
BreachCount: f.BreachCount,
|
||||
}
|
||||
}
|
||||
|
||||
enc := json.NewEncoder(w)
|
||||
enc.SetIndent("", " ")
|
||||
return enc.Encode(output)
|
||||
}
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
// ©AngelaMos | 2026
|
||||
// reporter.go
|
||||
|
||||
package reporter
|
||||
|
||||
import (
|
||||
"io"
|
||||
|
||||
"github.com/CarterPerez-dev/portia/pkg/types"
|
||||
)
|
||||
|
||||
type Reporter interface {
|
||||
Report(w io.Writer, result *types.ScanResult) error
|
||||
}
|
||||
|
||||
func New(format string) Reporter {
|
||||
switch format {
|
||||
case "json":
|
||||
return &JSON{}
|
||||
case "sarif":
|
||||
return &SARIF{}
|
||||
default:
|
||||
return &Terminal{}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,215 @@
|
|||
// ©AngelaMos | 2026
|
||||
// reporter_test.go
|
||||
|
||||
package reporter
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/CarterPerez-dev/portia/pkg/types"
|
||||
)
|
||||
|
||||
func testResult() *types.ScanResult {
|
||||
return &types.ScanResult{
|
||||
Findings: []types.Finding{
|
||||
{ //nolint:gosec
|
||||
RuleID: "aws-access-key-id",
|
||||
Description: "AWS Access Key ID",
|
||||
Severity: types.SeverityCritical,
|
||||
Match: `AKIAIOSFODNN7EXAMPLE`,
|
||||
Secret: "AKIAIOSFODNN7EXAMPLE",
|
||||
FilePath: "src/config.py",
|
||||
LineNumber: 42,
|
||||
LineContent: `aws_key = "AKIAIOSFODNN7EXAMPLE"`,
|
||||
CommitSHA: "abc123def456",
|
||||
Author: "dev@example.com",
|
||||
HIBPStatus: types.HIBPBreached,
|
||||
BreachCount: 1500,
|
||||
},
|
||||
{ //nolint:gosec
|
||||
RuleID: "generic-password",
|
||||
Description: "Password in Assignment",
|
||||
Severity: types.SeverityHigh,
|
||||
Match: `password = "xK9mP2vL5nQ8"`,
|
||||
Secret: "xK9mP2vL5nQ8",
|
||||
Entropy: 4.12,
|
||||
FilePath: "config.yaml",
|
||||
LineNumber: 10,
|
||||
LineContent: `password = "xK9mP2vL5nQ8"`,
|
||||
HIBPStatus: types.HIBPClean,
|
||||
},
|
||||
},
|
||||
TotalFiles: 100,
|
||||
TotalRules: 50,
|
||||
TotalFindings: 2,
|
||||
HIBPChecked: 2,
|
||||
HIBPBreached: 1,
|
||||
Duration: 1500 * time.Millisecond,
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewReporter(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
assert.IsType(t, &Terminal{}, New("terminal"))
|
||||
assert.IsType(t, &Terminal{}, New(""))
|
||||
assert.IsType(t, &JSON{}, New("json"))
|
||||
assert.IsType(t, &SARIF{}, New("sarif"))
|
||||
}
|
||||
|
||||
func TestTerminalReportWithFindings(t *testing.T) {
|
||||
t.Parallel()
|
||||
var buf bytes.Buffer
|
||||
r := &Terminal{}
|
||||
err := r.Report(&buf, testResult())
|
||||
require.NoError(t, err)
|
||||
|
||||
output := buf.String()
|
||||
assert.Contains(t, output, "2 secret(s) detected")
|
||||
assert.Contains(t, output, "aws-access-key-id")
|
||||
assert.Contains(t, output, "src/config.py")
|
||||
assert.Contains(t, output, "FOUND IN BREACHES")
|
||||
assert.Contains(t, output, "not found in breaches")
|
||||
assert.Contains(t, output, "4.12")
|
||||
assert.Contains(t, output, "abc123de")
|
||||
}
|
||||
|
||||
func TestTerminalReportNoFindings(t *testing.T) {
|
||||
t.Parallel()
|
||||
var buf bytes.Buffer
|
||||
r := &Terminal{}
|
||||
result := &types.ScanResult{
|
||||
TotalRules: 50,
|
||||
Duration: 500 * time.Millisecond,
|
||||
}
|
||||
err := r.Report(&buf, result)
|
||||
require.NoError(t, err)
|
||||
|
||||
output := buf.String()
|
||||
assert.Contains(t, output, "No secrets detected")
|
||||
}
|
||||
|
||||
func TestJSONReport(t *testing.T) {
|
||||
t.Parallel()
|
||||
var buf bytes.Buffer
|
||||
r := &JSON{}
|
||||
err := r.Report(&buf, testResult())
|
||||
require.NoError(t, err)
|
||||
|
||||
var output jsonOutput
|
||||
require.NoError(t, json.Unmarshal(buf.Bytes(), &output))
|
||||
|
||||
assert.Len(t, output.Findings, 2)
|
||||
assert.Equal(t, "aws-access-key-id", output.Findings[0].RuleID)
|
||||
assert.Equal(t, "CRITICAL", output.Findings[0].Severity)
|
||||
assert.NotContains(t, output.Findings[0].Secret, "AKIAIOSFODNN7EXAMPLE")
|
||||
assert.Equal(t, 100, output.Summary.TotalFiles)
|
||||
assert.Equal(t, 2, output.Summary.TotalFindings)
|
||||
assert.Equal(t, 50, output.Summary.TotalRules)
|
||||
}
|
||||
|
||||
func TestJSONReportNoFindings(t *testing.T) {
|
||||
t.Parallel()
|
||||
var buf bytes.Buffer
|
||||
r := &JSON{}
|
||||
result := &types.ScanResult{TotalRules: 50}
|
||||
err := r.Report(&buf, result)
|
||||
require.NoError(t, err)
|
||||
|
||||
var output jsonOutput
|
||||
require.NoError(t, json.Unmarshal(buf.Bytes(), &output))
|
||||
assert.Empty(t, output.Findings)
|
||||
}
|
||||
|
||||
func TestSARIFReport(t *testing.T) {
|
||||
t.Parallel()
|
||||
var buf bytes.Buffer
|
||||
r := &SARIF{}
|
||||
err := r.Report(&buf, testResult())
|
||||
require.NoError(t, err)
|
||||
|
||||
var log sarifLog
|
||||
require.NoError(t, json.Unmarshal(buf.Bytes(), &log))
|
||||
|
||||
assert.Equal(t, "2.1.0", log.Version)
|
||||
require.Len(t, log.Runs, 1)
|
||||
|
||||
run := log.Runs[0]
|
||||
assert.Equal(t, "portia", run.Tool.Driver.Name)
|
||||
assert.Len(t, run.Results, 2)
|
||||
|
||||
assert.Equal(t, "aws-access-key-id", run.Results[0].RuleID)
|
||||
assert.Equal(t, "error", run.Results[0].Level)
|
||||
assert.Equal(t,
|
||||
"src/config.py",
|
||||
run.Results[0].Locations[0].PhysicalLocation.
|
||||
ArtifactLocation.URI,
|
||||
)
|
||||
assert.Equal(t,
|
||||
42,
|
||||
run.Results[0].Locations[0].PhysicalLocation.Region.StartLine,
|
||||
)
|
||||
}
|
||||
|
||||
func TestSARIFReportNoFindings(t *testing.T) {
|
||||
t.Parallel()
|
||||
var buf bytes.Buffer
|
||||
r := &SARIF{}
|
||||
result := &types.ScanResult{TotalRules: 50}
|
||||
err := r.Report(&buf, result)
|
||||
require.NoError(t, err)
|
||||
|
||||
var log sarifLog
|
||||
require.NoError(t, json.Unmarshal(buf.Bytes(), &log))
|
||||
assert.Empty(t, log.Runs[0].Results)
|
||||
}
|
||||
|
||||
func TestMaskSecret(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := map[string]struct {
|
||||
input string
|
||||
want string
|
||||
}{
|
||||
"short secret": {
|
||||
input: "abc",
|
||||
want: "***",
|
||||
},
|
||||
"medium secret": {
|
||||
input: "AKIAIOSFODNN7EXA",
|
||||
want: "AKIA********7EXA",
|
||||
},
|
||||
"long secret": { //nolint:gosec
|
||||
input: "sk_live_4eC39HqLyjWDarjtT1zdp7dc",
|
||||
want: "sk_liv********************zdp7dc",
|
||||
},
|
||||
}
|
||||
|
||||
for name, tc := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
got := maskSecret(tc.input)
|
||||
assert.Equal(t, tc.want, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTruncateSHA(t *testing.T) {
|
||||
t.Parallel()
|
||||
assert.Equal(t, "abc123de", truncateSHA("abc123def456"))
|
||||
assert.Equal(t, "short", truncateSHA("short"))
|
||||
}
|
||||
|
||||
func TestSeverityToSARIF(t *testing.T) {
|
||||
t.Parallel()
|
||||
assert.Equal(t, "error", severityToSARIF(types.SeverityCritical))
|
||||
assert.Equal(t, "error", severityToSARIF(types.SeverityHigh))
|
||||
assert.Equal(t, "warning", severityToSARIF(types.SeverityMedium))
|
||||
assert.Equal(t, "note", severityToSARIF(types.SeverityLow))
|
||||
}
|
||||
|
|
@ -0,0 +1,168 @@
|
|||
// ©AngelaMos | 2026
|
||||
// sarif.go
|
||||
|
||||
package reporter
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
|
||||
"github.com/CarterPerez-dev/portia/pkg/types"
|
||||
)
|
||||
|
||||
type SARIF struct{}
|
||||
|
||||
type sarifLog struct {
|
||||
Version string `json:"version"`
|
||||
Schema string `json:"$schema"`
|
||||
Runs []sarifRun `json:"runs"`
|
||||
}
|
||||
|
||||
type sarifRun struct {
|
||||
Tool sarifTool `json:"tool"`
|
||||
Results []sarifResult `json:"results"`
|
||||
}
|
||||
|
||||
type sarifTool struct {
|
||||
Driver sarifDriver `json:"driver"`
|
||||
}
|
||||
|
||||
type sarifDriver struct {
|
||||
Name string `json:"name"`
|
||||
Version string `json:"version"`
|
||||
Rules []sarifRule `json:"rules,omitempty"`
|
||||
}
|
||||
|
||||
type sarifRule struct {
|
||||
ID string `json:"id"`
|
||||
ShortDescription sarifMessage `json:"shortDescription"`
|
||||
DefaultConfig sarifRuleConfig `json:"defaultConfiguration"`
|
||||
}
|
||||
|
||||
type sarifRuleConfig struct {
|
||||
Level string `json:"level"`
|
||||
}
|
||||
|
||||
type sarifMessage struct {
|
||||
Text string `json:"text"`
|
||||
}
|
||||
|
||||
type sarifResult struct {
|
||||
RuleID string `json:"ruleId"`
|
||||
Level string `json:"level"`
|
||||
Message sarifMessage `json:"message"`
|
||||
Locations []sarifLocation `json:"locations"`
|
||||
Properties map[string]any `json:"properties,omitempty"`
|
||||
}
|
||||
|
||||
type sarifLocation struct {
|
||||
PhysicalLocation sarifPhysicalLocation `json:"physicalLocation"`
|
||||
}
|
||||
|
||||
type sarifPhysicalLocation struct {
|
||||
ArtifactLocation sarifArtifactLocation `json:"artifactLocation"`
|
||||
Region sarifRegion `json:"region"`
|
||||
}
|
||||
|
||||
type sarifArtifactLocation struct {
|
||||
URI string `json:"uri"`
|
||||
}
|
||||
|
||||
type sarifRegion struct {
|
||||
StartLine int `json:"startLine"`
|
||||
}
|
||||
|
||||
func (s *SARIF) Report(
|
||||
w io.Writer, result *types.ScanResult,
|
||||
) error {
|
||||
ruleMap := make(map[string]bool)
|
||||
var sarifRules []sarifRule
|
||||
|
||||
for _, f := range result.Findings {
|
||||
if ruleMap[f.RuleID] {
|
||||
continue
|
||||
}
|
||||
ruleMap[f.RuleID] = true
|
||||
sarifRules = append(sarifRules, sarifRule{
|
||||
ID: f.RuleID,
|
||||
ShortDescription: sarifMessage{Text: f.Description},
|
||||
DefaultConfig: sarifRuleConfig{
|
||||
Level: severityToSARIF(f.Severity),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
var results []sarifResult
|
||||
for _, f := range result.Findings {
|
||||
r := sarifResult{
|
||||
RuleID: f.RuleID,
|
||||
Level: severityToSARIF(f.Severity),
|
||||
Message: sarifMessage{Text: f.Description},
|
||||
Locations: []sarifLocation{
|
||||
{
|
||||
PhysicalLocation: sarifPhysicalLocation{
|
||||
ArtifactLocation: sarifArtifactLocation{
|
||||
URI: f.FilePath,
|
||||
},
|
||||
Region: sarifRegion{
|
||||
StartLine: f.LineNumber,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
props := make(map[string]any)
|
||||
if f.CommitSHA != "" {
|
||||
props["commit"] = f.CommitSHA
|
||||
}
|
||||
if f.Entropy > 0 {
|
||||
props["entropy"] = f.Entropy
|
||||
}
|
||||
if f.HIBPStatus == types.HIBPBreached {
|
||||
props["hibp_breached"] = true
|
||||
props["breach_count"] = f.BreachCount
|
||||
}
|
||||
if len(props) > 0 {
|
||||
r.Properties = props
|
||||
}
|
||||
|
||||
results = append(results, r)
|
||||
}
|
||||
|
||||
log := sarifLog{
|
||||
Version: "2.1.0",
|
||||
Schema: "https://raw.githubusercontent.com/" +
|
||||
"oasis-tcs/sarif-spec/main/sarif-2.1/" +
|
||||
"schema/sarif-schema-2.1.0.json",
|
||||
Runs: []sarifRun{
|
||||
{
|
||||
Tool: sarifTool{
|
||||
Driver: sarifDriver{
|
||||
Name: "portia",
|
||||
Version: "1.0.0",
|
||||
Rules: sarifRules,
|
||||
},
|
||||
},
|
||||
Results: results,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
enc := json.NewEncoder(w)
|
||||
enc.SetIndent("", " ")
|
||||
return enc.Encode(log)
|
||||
}
|
||||
|
||||
func severityToSARIF(sev types.Severity) string {
|
||||
switch sev {
|
||||
case types.SeverityCritical, types.SeverityHigh:
|
||||
return "error"
|
||||
case types.SeverityMedium:
|
||||
return "warning"
|
||||
case types.SeverityLow:
|
||||
return "note"
|
||||
default:
|
||||
return "note"
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,219 @@
|
|||
// ©AngelaMos | 2026
|
||||
// terminal.go
|
||||
|
||||
package reporter
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/CarterPerez-dev/portia/internal/ui"
|
||||
"github.com/CarterPerez-dev/portia/pkg/types"
|
||||
)
|
||||
|
||||
type Terminal struct{}
|
||||
|
||||
type errWriter struct {
|
||||
w io.Writer
|
||||
err error
|
||||
}
|
||||
|
||||
func (ew *errWriter) printf(format string, args ...any) {
|
||||
if ew.err != nil {
|
||||
return
|
||||
}
|
||||
_, ew.err = fmt.Fprintf(ew.w, format, args...)
|
||||
}
|
||||
|
||||
func (ew *errWriter) println(args ...any) {
|
||||
if ew.err != nil {
|
||||
return
|
||||
}
|
||||
_, ew.err = fmt.Fprintln(ew.w, args...)
|
||||
}
|
||||
|
||||
func (t *Terminal) Report(
|
||||
w io.Writer, result *types.ScanResult,
|
||||
) error {
|
||||
ew := &errWriter{w: w}
|
||||
|
||||
if len(result.Findings) == 0 {
|
||||
ew.printf("\n%s %s\n\n",
|
||||
ui.Check,
|
||||
ui.Green("No secrets detected"),
|
||||
)
|
||||
t.printSummary(ew, result)
|
||||
return ew.err
|
||||
}
|
||||
|
||||
ew.printf("\n%s %s\n\n",
|
||||
ui.Warning,
|
||||
ui.RedBold(fmt.Sprintf(
|
||||
"%d secret(s) detected",
|
||||
len(result.Findings),
|
||||
)),
|
||||
)
|
||||
|
||||
sorted := make([]types.Finding, len(result.Findings))
|
||||
copy(sorted, result.Findings)
|
||||
sort.Slice(sorted, func(i, j int) bool {
|
||||
return sorted[i].Severity.Rank() < sorted[j].Severity.Rank()
|
||||
})
|
||||
|
||||
for i, f := range sorted {
|
||||
if i > 0 {
|
||||
ew.println()
|
||||
}
|
||||
t.printFinding(ew, &f)
|
||||
}
|
||||
|
||||
t.printSummary(ew, result)
|
||||
return ew.err
|
||||
}
|
||||
|
||||
func (t *Terminal) printFinding(
|
||||
ew *errWriter, f *types.Finding,
|
||||
) {
|
||||
sevColor := severityColor(f.Severity)
|
||||
|
||||
ew.printf("%s %s - %s\n",
|
||||
ui.HiGreen(ui.Diamond),
|
||||
sevColor(f.Severity.String()),
|
||||
ui.HiWhite(f.Description),
|
||||
)
|
||||
|
||||
ew.printf(" %s %s %s\n",
|
||||
ui.Green(ui.Arrow), ui.HiBlue("Rule: "), ui.Cyan(f.RuleID))
|
||||
ew.printf(" %s %s %s\n",
|
||||
ui.Green(ui.Arrow), ui.HiBlue("File: "),
|
||||
ui.WhiteBold(fmt.Sprintf("%s:%d", f.FilePath, f.LineNumber)))
|
||||
ew.printf(
|
||||
" %s %s %s\n",
|
||||
ui.Green(
|
||||
ui.Arrow,
|
||||
),
|
||||
ui.HiBlue("Secret: "),
|
||||
ui.YellowBold(maskSecret(f.Secret)),
|
||||
)
|
||||
|
||||
if f.Entropy > 0 {
|
||||
ew.printf(" %s %s %s\n",
|
||||
ui.Green(ui.Arrow), ui.HiBlue("Entropy: "),
|
||||
ui.HiGreenUnderline(fmt.Sprintf("%.2f bits", f.Entropy)))
|
||||
}
|
||||
|
||||
if f.CommitSHA != "" {
|
||||
ew.printf(" %s %s %s\n",
|
||||
ui.Green(ui.Arrow), ui.HiBlue("Commit: "),
|
||||
ui.Magenta(truncateSHA(f.CommitSHA)))
|
||||
}
|
||||
if f.Author != "" {
|
||||
ew.printf(" %s %s %s\n",
|
||||
ui.Green(ui.Arrow), ui.HiBlue("Author: "), f.Author)
|
||||
}
|
||||
|
||||
if f.HIBPStatus == types.HIBPBreached {
|
||||
ew.printf(" %s Breached: %s (%d occurrences)\n",
|
||||
ui.Warning,
|
||||
ui.RedBold("FOUND IN BREACHES"),
|
||||
f.BreachCount,
|
||||
)
|
||||
} else if f.HIBPStatus == types.HIBPClean {
|
||||
ew.printf(" %s HIBP: %s\n",
|
||||
ui.Shield, ui.Green("not found in breaches"))
|
||||
}
|
||||
|
||||
if f.LineContent != "" {
|
||||
ew.printf(" %s %s %s\n",
|
||||
ui.Green(ui.Arrow), ui.HiBlue("Context: "),
|
||||
ui.RedItalic(truncateLine(f.LineContent, 100)))
|
||||
}
|
||||
}
|
||||
|
||||
func (t *Terminal) printSummary(
|
||||
ew *errWriter, result *types.ScanResult,
|
||||
) {
|
||||
ew.printf("\n%s\n", ui.HRule(50))
|
||||
ew.printf(" %s %s\n",
|
||||
ui.HiBlueBold("Scan completed in"),
|
||||
ui.GreenItalic(result.Duration.Round(1e6).String()))
|
||||
ew.printf(" %s %s\n",
|
||||
ui.HiBlueBold("Files: "),
|
||||
ui.HiCyanItalic(fmt.Sprintf("%d", result.TotalFiles)))
|
||||
ew.printf(" %s %s\n",
|
||||
ui.HiBlueBold("Rules: "),
|
||||
ui.HiCyanItalic(fmt.Sprintf("%d", result.TotalRules)))
|
||||
ew.printf(" %s %s\n",
|
||||
ui.HiBlueBold("Findings:"),
|
||||
findingsColor(len(result.Findings)))
|
||||
|
||||
if result.HIBPChecked > 0 {
|
||||
ew.printf(" HIBP: %d checked, %s breached\n",
|
||||
result.HIBPChecked,
|
||||
breachColor(result.HIBPBreached))
|
||||
}
|
||||
ew.printf("%s\n\n", ui.HRule(50))
|
||||
}
|
||||
|
||||
func severityColor(
|
||||
sev types.Severity,
|
||||
) func(a ...interface{}) string {
|
||||
switch sev {
|
||||
case types.SeverityCritical:
|
||||
return ui.RedBold
|
||||
case types.SeverityHigh:
|
||||
return ui.Red
|
||||
case types.SeverityMedium:
|
||||
return ui.Yellow
|
||||
case types.SeverityLow:
|
||||
return ui.Blue
|
||||
default:
|
||||
return ui.Blue
|
||||
}
|
||||
}
|
||||
|
||||
func findingsColor(count int) string {
|
||||
s := fmt.Sprintf("%d", count)
|
||||
if count > 0 {
|
||||
return ui.RedBold(s)
|
||||
}
|
||||
return ui.Green(s)
|
||||
}
|
||||
|
||||
func breachColor(count int) string {
|
||||
s := fmt.Sprintf("%d", count)
|
||||
if count > 0 {
|
||||
return ui.RedBold(s)
|
||||
}
|
||||
return ui.Green(s)
|
||||
}
|
||||
|
||||
func maskSecret(secret string) string {
|
||||
if len(secret) <= 8 {
|
||||
return strings.Repeat("*", len(secret))
|
||||
}
|
||||
visible := 4
|
||||
if len(secret) > 20 {
|
||||
visible = 6
|
||||
}
|
||||
return secret[:visible] +
|
||||
strings.Repeat("*", len(secret)-visible*2) +
|
||||
secret[len(secret)-visible:]
|
||||
}
|
||||
|
||||
func truncateSHA(sha string) string {
|
||||
if len(sha) > 8 {
|
||||
return sha[:8]
|
||||
}
|
||||
return sha
|
||||
}
|
||||
|
||||
func truncateLine(line string, maxLen int) string {
|
||||
line = strings.TrimSpace(line)
|
||||
if len(line) <= maxLen {
|
||||
return line
|
||||
}
|
||||
return line[:maxLen-3] + "..."
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,541 @@
|
|||
// ©AngelaMos | 2026
|
||||
// builtin_test.go
|
||||
|
||||
package rules
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestRegisterBuiltins(t *testing.T) {
|
||||
t.Parallel()
|
||||
reg := NewRegistry()
|
||||
RegisterBuiltins(reg)
|
||||
assert.GreaterOrEqual(t, reg.Len(), 70)
|
||||
}
|
||||
|
||||
func TestBuiltinRuleIDs(t *testing.T) {
|
||||
t.Parallel()
|
||||
reg := NewRegistry()
|
||||
RegisterBuiltins(reg)
|
||||
seen := make(map[string]bool)
|
||||
for _, r := range reg.All() {
|
||||
assert.NotEmpty(t, r.ID)
|
||||
assert.NotEmpty(t, r.Description)
|
||||
assert.NotEmpty(t, r.Keywords)
|
||||
assert.NotNil(t, r.Pattern)
|
||||
assert.False(t, seen[r.ID], "duplicate ID: %s", r.ID)
|
||||
seen[r.ID] = true
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuiltinPatternMatches(t *testing.T) { //nolint:funlen,gocognit
|
||||
t.Parallel()
|
||||
|
||||
tests := map[string]struct {
|
||||
ruleID string
|
||||
input string
|
||||
wantMatch bool
|
||||
wantGroup string
|
||||
}{
|
||||
"aws access key": { //nolint:gosec
|
||||
ruleID: "aws-access-key-id",
|
||||
input: `aws_key = "AKIAIOSFODNN7EXAMPLE"`,
|
||||
wantMatch: true,
|
||||
wantGroup: "AKIAIOSFODNN7EXAMPLE",
|
||||
},
|
||||
"aws access key ABIA prefix": {
|
||||
ruleID: "aws-access-key-id",
|
||||
input: `ABIAIOSFODNN7EXAMPL0`,
|
||||
wantMatch: true,
|
||||
wantGroup: "ABIAIOSFODNN7EXAMPL0",
|
||||
},
|
||||
"aws access key too short": {
|
||||
ruleID: "aws-access-key-id",
|
||||
input: `AKIA1234`,
|
||||
wantMatch: false,
|
||||
},
|
||||
"github pat classic": {
|
||||
ruleID: "github-pat-classic",
|
||||
input: `ghp_ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghij`,
|
||||
wantMatch: true,
|
||||
wantGroup: "ghp_ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghij",
|
||||
},
|
||||
"github pat fine-grained": {
|
||||
ruleID: "github-pat-fine",
|
||||
input: `github_pat_abcdefghijABCDEFGHIJKL_abcdefghij0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789ABC`,
|
||||
wantMatch: true,
|
||||
},
|
||||
"github oauth": {
|
||||
ruleID: "github-oauth-token",
|
||||
input: `gho_ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghij`,
|
||||
wantMatch: true,
|
||||
wantGroup: "gho_ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghij",
|
||||
},
|
||||
"github app token": {
|
||||
ruleID: "github-app-token",
|
||||
input: `ghs_ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghij`,
|
||||
wantMatch: true,
|
||||
wantGroup: "ghs_ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghij",
|
||||
},
|
||||
"github refresh token": {
|
||||
ruleID: "github-refresh-token",
|
||||
input: `ghr_ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghij`,
|
||||
wantMatch: true,
|
||||
wantGroup: "ghr_ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghij",
|
||||
},
|
||||
"gitlab pat": {
|
||||
ruleID: "gitlab-pat",
|
||||
input: `glpat-xYz1234567890AbCdEfGh`,
|
||||
wantMatch: true,
|
||||
wantGroup: "glpat-xYz1234567890AbCdEfGh",
|
||||
},
|
||||
"gitlab pipeline trigger": {
|
||||
ruleID: "gitlab-pipeline-trigger",
|
||||
input: `glptt-abcdefghijklmnopqrstuvwxyz0123456789ABCD`,
|
||||
wantMatch: true,
|
||||
wantGroup: "glptt-abcdefghijklmnopqrstuvwxyz0123456789ABCD",
|
||||
},
|
||||
"gitlab runner token": {
|
||||
ruleID: "gitlab-runner-token",
|
||||
input: `glrt-xYz1234567890AbCdEfGh`,
|
||||
wantMatch: true,
|
||||
wantGroup: "glrt-xYz1234567890AbCdEfGh",
|
||||
},
|
||||
"gcp api key": {
|
||||
ruleID: "gcp-api-key",
|
||||
input: `AIzaSyDabcdefghij1234567890KLMNOPQRSTUV`,
|
||||
wantMatch: true,
|
||||
},
|
||||
"gcp service account": {
|
||||
ruleID: "gcp-service-account",
|
||||
input: `"type" : "service_account"`,
|
||||
wantMatch: true,
|
||||
},
|
||||
"gcp oauth secret": {
|
||||
ruleID: "gcp-oauth-client-secret",
|
||||
input: `GOCSPX-aBcDeFgHiJkLmNoPqRsTuVwXyZ01`,
|
||||
wantMatch: true,
|
||||
},
|
||||
"azure storage key": {
|
||||
ruleID: "azure-storage-key",
|
||||
input: `AccountKey=` +
|
||||
`abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz`,
|
||||
wantMatch: true,
|
||||
},
|
||||
"slack bot token": {
|
||||
ruleID: "slack-bot-token",
|
||||
input: `xoxb-1234567890-9876543210-AbCdEfGhIjKlMnOpQrStUvWx`,
|
||||
wantMatch: true,
|
||||
},
|
||||
"slack webhook": { //nolint:gosec
|
||||
ruleID: "slack-webhook",
|
||||
input: `https://hooks.slack.com/services/T01234567/B01234567/AbCdEfGhIjKlMnOpQrStUvWx`,
|
||||
wantMatch: true,
|
||||
},
|
||||
"stripe live key": { //nolint:gosec
|
||||
ruleID: "stripe-live-secret",
|
||||
input: `sk_live_4eC39HqLyjWDarjtT1zdp7dc`,
|
||||
wantMatch: true,
|
||||
wantGroup: "sk_live_4eC39HqLyjWDarjtT1zdp7dc",
|
||||
},
|
||||
"stripe test key": {
|
||||
ruleID: "stripe-test-secret",
|
||||
input: `sk_test_4eC39HqLyjWDarjtT1zdp7dc`, //nolint:gosec
|
||||
wantMatch: true,
|
||||
wantGroup: "sk_test_4eC39HqLyjWDarjtT1zdp7dc", //nolint:gosec
|
||||
},
|
||||
"stripe restricted live": { //nolint:gosec
|
||||
ruleID: "stripe-live-restricted",
|
||||
input: `rk_live_4eC39HqLyjWDarjtT1zdp7dc`,
|
||||
wantMatch: true,
|
||||
wantGroup: "rk_live_4eC39HqLyjWDarjtT1zdp7dc",
|
||||
},
|
||||
"stripe webhook secret": {
|
||||
ruleID: "stripe-webhook-secret",
|
||||
input: `whsec_MfKBGsXP8r7B2cGnQ9jT6KxL12AbCdEf`,
|
||||
wantMatch: true,
|
||||
},
|
||||
"twilio api key": {
|
||||
ruleID: "twilio-api-key",
|
||||
input: `SK1234567890abcdef1234567890abcdef`,
|
||||
wantMatch: true,
|
||||
wantGroup: "SK1234567890abcdef1234567890abcdef",
|
||||
},
|
||||
"twilio account sid": {
|
||||
ruleID: "twilio-account-sid",
|
||||
input: `AC1234567890abcdef1234567890abcdef`,
|
||||
wantMatch: true,
|
||||
wantGroup: "AC1234567890abcdef1234567890abcdef",
|
||||
},
|
||||
"sendgrid api key": {
|
||||
ruleID: "sendgrid-api-key",
|
||||
input: `SG.aBcDeFgHiJkLmNoPqRsTuw.xYzAbCdEfGhIjKlMnOpQrStUvWxYzAbCdEfGhIjKlMn`,
|
||||
wantMatch: true,
|
||||
},
|
||||
"shopify access token": {
|
||||
ruleID: "shopify-access-token",
|
||||
input: `shpat_abcdef0123456789abcdef0123456789`,
|
||||
wantMatch: true,
|
||||
wantGroup: "shpat_abcdef0123456789abcdef0123456789",
|
||||
},
|
||||
"shopify custom app": {
|
||||
ruleID: "shopify-custom-app",
|
||||
input: `shpca_abcdef0123456789abcdef0123456789`,
|
||||
wantMatch: true,
|
||||
wantGroup: "shpca_abcdef0123456789abcdef0123456789",
|
||||
},
|
||||
"shopify private app": {
|
||||
ruleID: "shopify-private-app",
|
||||
input: `shppa_abcdef0123456789abcdef0123456789`,
|
||||
wantMatch: true,
|
||||
wantGroup: "shppa_abcdef0123456789abcdef0123456789",
|
||||
},
|
||||
"shopify shared secret": {
|
||||
ruleID: "shopify-shared-secret",
|
||||
input: `shpss_abcdef0123456789abcdef0123456789`,
|
||||
wantMatch: true,
|
||||
wantGroup: "shpss_abcdef0123456789abcdef0123456789",
|
||||
},
|
||||
"npm access token": {
|
||||
ruleID: "npm-access-token",
|
||||
input: `npm_aBcDeFgHiJkLmNoPqRsTuVwXyZ0123456789`,
|
||||
wantMatch: true,
|
||||
},
|
||||
"pypi token": {
|
||||
ruleID: "pypi-api-token",
|
||||
input: `pypi-AgEIcHlwaS5vcmcCJDU5NTk5YTJhLWIwN2QtNDRkZi1iM` +
|
||||
`jIxLTk2OWU4YmViZDM3NgACKlszLCJjODA0ZWE1OC1kMjFiLTQzMjMtYWR` +
|
||||
`mNy0xZjQ4MGQ`,
|
||||
wantMatch: true,
|
||||
},
|
||||
"rubygems api key": {
|
||||
ruleID: "rubygems-api-key",
|
||||
input: `rubygems_abcdef0123456789abcdef0123456789abcdef0123456789`,
|
||||
wantMatch: true,
|
||||
},
|
||||
"docker hub pat": {
|
||||
ruleID: "docker-hub-pat",
|
||||
input: `dckr_pat_aBcDeFgHiJkLmNoPqRsTuVwXyZ01`,
|
||||
wantMatch: true,
|
||||
},
|
||||
"jwt": {
|
||||
ruleID: "jwt-token",
|
||||
input: `eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.` +
|
||||
`eyJzdWIiOiIxMjM0NTY3ODkwIn0.` +
|
||||
`dozjgNryP4J3jVmNHl0w5N_XgL0n3I9PlFUP0THsR8U`,
|
||||
wantMatch: true,
|
||||
},
|
||||
"rsa private key": { //nolint:gosec
|
||||
ruleID: "ssh-private-key-rsa",
|
||||
input: `-----BEGIN RSA PRIVATE KEY-----`,
|
||||
wantMatch: true,
|
||||
},
|
||||
"openssh private key": {
|
||||
ruleID: "ssh-private-key-openssh",
|
||||
input: `-----BEGIN OPENSSH PRIVATE KEY-----`, //nolint:gosec
|
||||
wantMatch: true,
|
||||
},
|
||||
"ec private key": { //nolint:gosec
|
||||
ruleID: "ssh-private-key-ec",
|
||||
input: `-----BEGIN EC PRIVATE KEY-----`,
|
||||
wantMatch: true,
|
||||
},
|
||||
"dsa private key": { //nolint:gosec
|
||||
ruleID: "ssh-private-key-dsa",
|
||||
input: `-----BEGIN DSA PRIVATE KEY-----`,
|
||||
wantMatch: true,
|
||||
},
|
||||
"pgp private key": { //nolint:gosec
|
||||
ruleID: "pgp-private-key",
|
||||
input: `-----BEGIN PGP PRIVATE KEY BLOCK-----`,
|
||||
wantMatch: true,
|
||||
},
|
||||
"pkcs8 private key": {
|
||||
ruleID: "private-key-pkcs8",
|
||||
input: `-----BEGIN PRIVATE KEY-----`,
|
||||
wantMatch: true,
|
||||
},
|
||||
"generic password": {
|
||||
ruleID: "generic-password",
|
||||
input: `password = "my$ecretP@ss99"`,
|
||||
wantMatch: true,
|
||||
wantGroup: "my$ecretP@ss99",
|
||||
},
|
||||
"generic password no match on short": {
|
||||
ruleID: "generic-password",
|
||||
input: `password = "abc"`,
|
||||
wantMatch: false,
|
||||
},
|
||||
"generic secret": {
|
||||
ruleID: "generic-secret",
|
||||
input: `secret_key = "xK9mP2vL5nQ8jR3tB7"`,
|
||||
wantMatch: true,
|
||||
wantGroup: "xK9mP2vL5nQ8jR3tB7",
|
||||
},
|
||||
"generic api key": {
|
||||
ruleID: "generic-api-key",
|
||||
input: `api_key = "aK9mP2vL5nQ8jR3tB7wX4cD"`,
|
||||
wantMatch: true,
|
||||
wantGroup: "aK9mP2vL5nQ8jR3tB7wX4cD",
|
||||
},
|
||||
"generic token": {
|
||||
ruleID: "generic-token",
|
||||
input: `access_token = "xK9mP2vL5nQ8jR3tB7wY"`,
|
||||
wantMatch: true,
|
||||
wantGroup: "xK9mP2vL5nQ8jR3tB7wY",
|
||||
},
|
||||
"postgres connection": { //nolint:gosec
|
||||
ruleID: "postgres-connection",
|
||||
input: `postgresql://admin:s3cr3t@db.example.com:5432/mydb`,
|
||||
wantMatch: true,
|
||||
},
|
||||
"mysql connection": { //nolint:gosec
|
||||
ruleID: "mysql-connection",
|
||||
input: `mysql://root:password123@127.0.0.1:3306/app`,
|
||||
wantMatch: true,
|
||||
},
|
||||
"mongodb connection": { //nolint:gosec
|
||||
ruleID: "mongodb-connection",
|
||||
input: `mongodb+srv://user:p4ssw0rd@cluster0.abc.mongodb.net/db`,
|
||||
wantMatch: true,
|
||||
},
|
||||
"redis connection": { //nolint:gosec
|
||||
ruleID: "redis-connection",
|
||||
input: `redis://default:s3cret@redis.example.com:6379/0`,
|
||||
wantMatch: true,
|
||||
},
|
||||
"hashicorp vault token": {
|
||||
ruleID: "hashicorp-vault-token",
|
||||
input: `hvs.CAESIGH3YzJfaBcDeFgHiJkLmNoPqR`,
|
||||
wantMatch: true,
|
||||
},
|
||||
"digitalocean pat": {
|
||||
ruleID: "digitalocean-pat",
|
||||
input: `dop_v1_abcdef01234567890abcdef01234567890abcdef01234567890abcdef0123456`,
|
||||
wantMatch: true,
|
||||
wantGroup: "dop_v1_abcdef01234567890abcdef01234567890abcdef01234567890abcdef0123456",
|
||||
},
|
||||
"linear api key": {
|
||||
ruleID: "linear-api-key",
|
||||
input: `lin_api_aBcDeFgHiJkLmNoPqRsTuVwXyZaBcDeFgHiJkLmN`,
|
||||
wantMatch: true,
|
||||
},
|
||||
"age secret key": {
|
||||
ruleID: "age-secret-key",
|
||||
input: `AGE-SECRET-KEY-1QQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQ`,
|
||||
wantMatch: true,
|
||||
},
|
||||
"doppler token": {
|
||||
ruleID: "doppler-token",
|
||||
input: `dp.st.aBcDeFgHiJkLmNoPqRsTuVwXyZaBcDeFgHiJkLmN`,
|
||||
wantMatch: true,
|
||||
},
|
||||
"grafana sa token": {
|
||||
ruleID: "grafana-api-key",
|
||||
input: `glsa_aBcDeFgHiJkLmNoPqRsTuVwXyZaBcDeF_12345678`,
|
||||
wantMatch: true,
|
||||
},
|
||||
"grafana cloud token": {
|
||||
ruleID: "grafana-cloud-token",
|
||||
input: `glc_aBcDeFgHiJkLmNoPqRsTuVwXyZaBcDeF1`,
|
||||
wantMatch: true,
|
||||
},
|
||||
"databricks token": {
|
||||
ruleID: "databricks-token",
|
||||
input: `dapi1234567890abcdef1234567890abcdef`,
|
||||
wantMatch: true,
|
||||
wantGroup: "dapi1234567890abcdef1234567890abcdef",
|
||||
},
|
||||
"huggingface token": {
|
||||
ruleID: "huggingface-token",
|
||||
input: `hf_aBcDeFgHiJkLmNoPqRsTuVwXyZaBcDeFgHiJ`,
|
||||
wantMatch: true,
|
||||
wantGroup: "hf_aBcDeFgHiJkLmNoPqRsTuVwXyZaBcDeFgHiJ",
|
||||
},
|
||||
"netlify token": {
|
||||
ruleID: "netlify-token",
|
||||
input: `nfp_aBcDeFgHiJkLmNoPqRsTuVwXyZaBcDeFgHiJkLmN`,
|
||||
wantMatch: true,
|
||||
},
|
||||
"postman api key": {
|
||||
ruleID: "postman-api-key",
|
||||
input: `PMAK-abcdef0123456789abcdef01-abcdef0123456789abcdef0123456789ab`,
|
||||
wantMatch: true,
|
||||
},
|
||||
"figma pat": {
|
||||
ruleID: "figma-pat",
|
||||
input: `figd_aBcDeFgHiJkLmNoPqRsTuVwXyZaBcDeFgHiJkLmN`,
|
||||
wantMatch: true,
|
||||
},
|
||||
"flyio token": {
|
||||
ruleID: "flyio-token",
|
||||
input: `fo1_aBcDeFgHiJkLmNoPqRsTuVwXyZaBcDeFgHiJkLmN`,
|
||||
wantMatch: true,
|
||||
},
|
||||
"planetscale token": {
|
||||
ruleID: "planetscale-token",
|
||||
input: `pscale_tkn_aBcDeFgHiJkLmNoPqRsTuVwXyZaBcDeFgHiJkLmN`,
|
||||
wantMatch: true,
|
||||
},
|
||||
"replicate token": {
|
||||
ruleID: "replicate-api-token",
|
||||
input: `r8_aBcDeFgHiJkLmNoPqRsTuVwXyZaBcDeFgHiJkLmNoPqR`,
|
||||
wantMatch: true,
|
||||
},
|
||||
"sentry auth token": {
|
||||
ruleID: "sentry-auth-token",
|
||||
input: `sntrys_aBcDeFgHiJkLmNoPqRsTuVwXyZaBcDeFgHiJkLmNoPqRsTuVwXyZaBcDeFgHi`,
|
||||
wantMatch: true,
|
||||
},
|
||||
"atlassian token": {
|
||||
ruleID: "atlassian-api-token",
|
||||
input: `ATATTaBcDeFgHiJkLmNoPqRsTuVwXyZaBcDeFgHiJkLmNoPqRsTuVwX`,
|
||||
wantMatch: true,
|
||||
},
|
||||
"render api key": {
|
||||
ruleID: "render-api-key",
|
||||
input: `rnd_aBcDeFgHiJkLmNoPqRsTuVwXyZaBcDeFgHi`,
|
||||
wantMatch: true,
|
||||
},
|
||||
"launchdarkly sdk key": {
|
||||
ruleID: "launchdarkly-sdk-key",
|
||||
input: `sdk-a1b2c3d4-e5f6-7890-abcd-ef1234567890`,
|
||||
wantMatch: true,
|
||||
wantGroup: "sdk-a1b2c3d4-e5f6-7890-abcd-ef1234567890",
|
||||
},
|
||||
"launchdarkly api key": {
|
||||
ruleID: "launchdarkly-api-key",
|
||||
input: `api-a1b2c3d4-e5f6-7890-abcd-ef1234567890`,
|
||||
wantMatch: true,
|
||||
wantGroup: "api-a1b2c3d4-e5f6-7890-abcd-ef1234567890",
|
||||
},
|
||||
"okta api token": {
|
||||
ruleID: "okta-api-token",
|
||||
input: `OKTA_API_TOKEN = "00ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijkl"`,
|
||||
wantMatch: true,
|
||||
wantGroup: "00ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijkl",
|
||||
},
|
||||
"okta ssws header": {
|
||||
ruleID: "okta-api-token",
|
||||
input: `SSWS = "00xK9mP2vL5nQ8jR3tB7wX4cD6eF8gH0iJ2kL"`,
|
||||
wantMatch: true,
|
||||
wantGroup: "00xK9mP2vL5nQ8jR3tB7wX4cD6eF8gH0iJ2kL",
|
||||
},
|
||||
"supabase service key": {
|
||||
ruleID: "supabase-service-key",
|
||||
input: `SUPABASE_SERVICE_ROLE_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJvbGUiOiJzZXJ2aWNlX3JvbGUifQ.xxxxxxxx`,
|
||||
wantMatch: true,
|
||||
wantGroup: "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJvbGUiOiJzZXJ2aWNlX3JvbGUifQ.xxxxxxxx",
|
||||
},
|
||||
"supabase service key quoted": {
|
||||
ruleID: "supabase-service-key",
|
||||
input: `SUPABASE_SERVICE_ROLE_KEY="eyJhbGciOiJIUzI1NiJ9.eyJyb2xlIjoic2VydmljZV9yb2xlIn0.sig123"`,
|
||||
wantMatch: true,
|
||||
},
|
||||
"no match on normal text": {
|
||||
ruleID: "aws-access-key-id",
|
||||
input: `this is just normal code`,
|
||||
wantMatch: false,
|
||||
},
|
||||
}
|
||||
|
||||
reg := NewRegistry()
|
||||
RegisterBuiltins(reg)
|
||||
|
||||
for name, tc := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
rule, ok := reg.Get(tc.ruleID)
|
||||
require.True(t, ok, "rule %s not found", tc.ruleID)
|
||||
|
||||
match := rule.Pattern.FindStringSubmatch(tc.input)
|
||||
if tc.wantMatch {
|
||||
require.NotNil(t, match,
|
||||
"expected pattern match for rule %s", tc.ruleID)
|
||||
if tc.wantGroup != "" && rule.SecretGroup > 0 {
|
||||
require.Greater(t, len(match), rule.SecretGroup)
|
||||
assert.Equal(t, tc.wantGroup,
|
||||
match[rule.SecretGroup])
|
||||
}
|
||||
} else {
|
||||
assert.Nil(t, match,
|
||||
"unexpected match for rule %s", tc.ruleID)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuiltinKeywordMatches(t *testing.T) {
|
||||
t.Parallel()
|
||||
reg := NewRegistry()
|
||||
RegisterBuiltins(reg)
|
||||
|
||||
tests := map[string]struct {
|
||||
content string
|
||||
wantIDs []string
|
||||
}{
|
||||
"aws content": { //nolint:gosec
|
||||
content: "found AKIAIOSFODNN7EXAMPLE in config",
|
||||
wantIDs: []string{"aws-access-key-id"},
|
||||
},
|
||||
"github content": {
|
||||
content: "token = ghp_abc123",
|
||||
wantIDs: []string{"github-pat-classic"},
|
||||
},
|
||||
"stripe content": {
|
||||
content: "key = sk_live_abc123",
|
||||
wantIDs: []string{"stripe-live-secret"},
|
||||
},
|
||||
"private key content": { //nolint:gosec
|
||||
content: "-----BEGIN RSA PRIVATE KEY-----",
|
||||
wantIDs: []string{"ssh-private-key-rsa"},
|
||||
},
|
||||
"postgres content": { //nolint:gosec
|
||||
content: "DATABASE_URL=postgres://user:pass@host/db",
|
||||
wantIDs: []string{"postgres-connection"},
|
||||
},
|
||||
}
|
||||
|
||||
for name, tc := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
matched := reg.MatchKeywords(tc.content)
|
||||
ids := make([]string, len(matched))
|
||||
for i, r := range matched {
|
||||
ids[i] = r.ID
|
||||
}
|
||||
for _, wantID := range tc.wantIDs {
|
||||
assert.Contains(t, ids, wantID)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuiltinNoFalsePositives(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
reg := NewRegistry()
|
||||
RegisterBuiltins(reg)
|
||||
|
||||
benignInputs := []string{
|
||||
`import os`,
|
||||
`func main() {}`,
|
||||
`const x = 42`,
|
||||
`// this is a comment`,
|
||||
`print("hello world")`,
|
||||
`if err != nil { return err }`,
|
||||
`for i := 0; i < 10; i++ {}`,
|
||||
}
|
||||
|
||||
for _, rule := range reg.All() {
|
||||
for _, input := range benignInputs {
|
||||
match := rule.Pattern.FindString(input)
|
||||
assert.Empty(t, match,
|
||||
"rule %s false positive on: %s", rule.ID, input)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,168 @@
|
|||
// ©AngelaMos | 2026
|
||||
// entropy.go
|
||||
|
||||
package rules
|
||||
|
||||
import (
|
||||
"math"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
Base64Charset = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=-_"
|
||||
HexCharset = "0123456789abcdefABCDEF"
|
||||
AlphanumericCharset = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
|
||||
)
|
||||
|
||||
var (
|
||||
base64Set = buildCharsetSet(Base64Charset)
|
||||
hexSet = buildCharsetSet(HexCharset)
|
||||
alphanumericSet = buildCharsetSet(AlphanumericCharset)
|
||||
)
|
||||
|
||||
type EntropyToken struct {
|
||||
Value string
|
||||
Entropy float64
|
||||
}
|
||||
|
||||
func ShannonEntropy(data, charset string) float64 {
|
||||
if len(data) == 0 {
|
||||
return 0.0
|
||||
}
|
||||
|
||||
filtered := data
|
||||
if charset != "" {
|
||||
cs := charsetSet(charset)
|
||||
var b strings.Builder
|
||||
b.Grow(len(data))
|
||||
for _, c := range data {
|
||||
if cs[c] {
|
||||
b.WriteRune(c)
|
||||
}
|
||||
}
|
||||
filtered = b.String()
|
||||
if len(filtered) == 0 {
|
||||
return 0.0
|
||||
}
|
||||
}
|
||||
|
||||
freq := make(map[rune]int)
|
||||
for _, c := range filtered {
|
||||
freq[c]++
|
||||
}
|
||||
|
||||
length := float64(len([]rune(filtered)))
|
||||
entropy := 0.0
|
||||
for _, count := range freq {
|
||||
p := float64(count) / length
|
||||
if p > 0 {
|
||||
entropy -= p * math.Log2(p)
|
||||
}
|
||||
}
|
||||
return entropy
|
||||
}
|
||||
|
||||
func DetectCharset(s string) string {
|
||||
hexCount := 0
|
||||
b64Count := 0
|
||||
total := 0
|
||||
|
||||
for _, c := range s {
|
||||
total++
|
||||
if hexSet[c] {
|
||||
hexCount++
|
||||
}
|
||||
if base64Set[c] {
|
||||
b64Count++
|
||||
}
|
||||
}
|
||||
|
||||
if total == 0 {
|
||||
return "alphanumeric"
|
||||
}
|
||||
|
||||
hexRatio := float64(hexCount) / float64(total)
|
||||
b64Ratio := float64(b64Count) / float64(total)
|
||||
|
||||
if hexRatio == 1.0 && isAllHexChars(s) {
|
||||
return "hex"
|
||||
}
|
||||
if b64Ratio >= 0.95 {
|
||||
return "base64"
|
||||
}
|
||||
return "alphanumeric"
|
||||
}
|
||||
|
||||
func ExtractHighEntropyTokens(
|
||||
line string,
|
||||
charset string,
|
||||
threshold float64,
|
||||
minLen int,
|
||||
) []EntropyToken {
|
||||
cs := charsetSet(charset)
|
||||
var tokens []EntropyToken
|
||||
var current []rune
|
||||
|
||||
for _, c := range line {
|
||||
if cs[c] {
|
||||
current = append(current, c)
|
||||
} else {
|
||||
if len(current) >= minLen {
|
||||
token := string(current)
|
||||
ent := ShannonEntropy(token, charset)
|
||||
if ent >= threshold {
|
||||
tokens = append(tokens, EntropyToken{
|
||||
Value: token,
|
||||
Entropy: math.Round(ent*1000) / 1000,
|
||||
})
|
||||
}
|
||||
}
|
||||
current = current[:0]
|
||||
}
|
||||
}
|
||||
|
||||
if len(current) >= minLen {
|
||||
token := string(current)
|
||||
ent := ShannonEntropy(token, charset)
|
||||
if ent >= threshold {
|
||||
tokens = append(tokens, EntropyToken{
|
||||
Value: token,
|
||||
Entropy: math.Round(ent*1000) / 1000,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return tokens
|
||||
}
|
||||
|
||||
func charsetSet(charset string) map[rune]bool {
|
||||
switch charset {
|
||||
case Base64Charset:
|
||||
return base64Set
|
||||
case HexCharset:
|
||||
return hexSet
|
||||
case AlphanumericCharset:
|
||||
return alphanumericSet
|
||||
default:
|
||||
return buildCharsetSet(charset)
|
||||
}
|
||||
}
|
||||
|
||||
func buildCharsetSet(charset string) map[rune]bool {
|
||||
s := make(map[rune]bool, len(charset))
|
||||
for _, c := range charset {
|
||||
s[c] = true
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func isAllHexChars(s string) bool {
|
||||
for _, c := range s {
|
||||
if !((c >= '0' && c <= '9') ||
|
||||
(c >= 'a' && c <= 'f') ||
|
||||
(c >= 'A' && c <= 'F')) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
|
@ -0,0 +1,226 @@
|
|||
// ©AngelaMos | 2026
|
||||
// entropy_test.go
|
||||
|
||||
package rules
|
||||
|
||||
import (
|
||||
"math"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestShannonEntropy(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := map[string]struct {
|
||||
data string
|
||||
charset string
|
||||
wantMin float64
|
||||
wantMax float64
|
||||
}{
|
||||
"empty string": {
|
||||
data: "",
|
||||
charset: "",
|
||||
wantMin: 0.0,
|
||||
wantMax: 0.0,
|
||||
},
|
||||
"single repeated character": {
|
||||
data: "aaaaaaa",
|
||||
charset: "",
|
||||
wantMin: 0.0,
|
||||
wantMax: 0.0,
|
||||
},
|
||||
"two equally distributed chars": {
|
||||
data: "ab",
|
||||
charset: "",
|
||||
wantMin: 1.0,
|
||||
wantMax: 1.0,
|
||||
},
|
||||
"four equally distributed chars": {
|
||||
data: "abcd",
|
||||
charset: "",
|
||||
wantMin: 2.0,
|
||||
wantMax: 2.0,
|
||||
},
|
||||
"real password low entropy": {
|
||||
data: "password123",
|
||||
charset: "",
|
||||
wantMin: 2.5,
|
||||
wantMax: 3.5,
|
||||
},
|
||||
"aws key high entropy": { //nolint:gosec
|
||||
data: "AKIAIOSFODNN7EXAMPLE",
|
||||
charset: Base64Charset,
|
||||
wantMin: 3.5,
|
||||
wantMax: 5.0,
|
||||
},
|
||||
"hex string": {
|
||||
data: "a1b2c3d4e5f6a7b8c9d0",
|
||||
charset: HexCharset,
|
||||
wantMin: 3.0,
|
||||
wantMax: 4.0,
|
||||
},
|
||||
"high entropy random base64": {
|
||||
data: "kR9mPx2vBnQ8jL5wYz3hTf",
|
||||
charset: Base64Charset,
|
||||
wantMin: 4.0,
|
||||
wantMax: 5.5,
|
||||
},
|
||||
"charset filtering removes non-charset": {
|
||||
data: "abc!!!def!!!ghi",
|
||||
charset: AlphanumericCharset,
|
||||
wantMin: 2.5,
|
||||
wantMax: 3.5,
|
||||
},
|
||||
"empty after charset filter": {
|
||||
data: "!!!???",
|
||||
charset: AlphanumericCharset,
|
||||
wantMin: 0.0,
|
||||
wantMax: 0.0,
|
||||
},
|
||||
}
|
||||
|
||||
for name, tc := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
got := ShannonEntropy(tc.data, tc.charset)
|
||||
assert.GreaterOrEqual(t, got, tc.wantMin,
|
||||
"entropy too low")
|
||||
assert.LessOrEqual(t, got, tc.wantMax,
|
||||
"entropy too high")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestShannonEntropyExactValues(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
got := ShannonEntropy("ab", "")
|
||||
assert.InDelta(t, 1.0, got, 0.001)
|
||||
|
||||
got = ShannonEntropy("aabb", "")
|
||||
assert.InDelta(t, 1.0, got, 0.001)
|
||||
|
||||
got = ShannonEntropy("abcdefgh", "")
|
||||
assert.InDelta(t, 3.0, got, 0.001)
|
||||
}
|
||||
|
||||
func TestDetectCharset(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := map[string]struct {
|
||||
input string
|
||||
want string
|
||||
}{
|
||||
"hex string": {
|
||||
input: "a1b2c3d4e5f6",
|
||||
want: "hex",
|
||||
},
|
||||
"base64 with special chars": {
|
||||
input: "kR9mPx2vBnQ8jL5w+/YzTf==",
|
||||
want: "base64",
|
||||
},
|
||||
"pure alphanumeric": {
|
||||
input: "HelloWorld123XYZ",
|
||||
want: "base64",
|
||||
},
|
||||
"mixed with symbols": {
|
||||
input: "hello@world#foo$bar",
|
||||
want: "alphanumeric",
|
||||
},
|
||||
"empty string": {
|
||||
input: "",
|
||||
want: "alphanumeric",
|
||||
},
|
||||
"only hex digits": {
|
||||
input: "deadbeef0123456789",
|
||||
want: "hex",
|
||||
},
|
||||
}
|
||||
|
||||
for name, tc := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
got := DetectCharset(tc.input)
|
||||
assert.Equal(t, tc.want, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractHighEntropyTokens(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := map[string]struct {
|
||||
line string
|
||||
charset string
|
||||
threshold float64
|
||||
minLen int
|
||||
wantCount int
|
||||
wantFirst string
|
||||
}{
|
||||
"extracts high entropy token": {
|
||||
line: `api_key = "kR9mPx2vBnQ8jL5wYz3hTf6aU7cD"`,
|
||||
charset: Base64Charset,
|
||||
threshold: 3.5,
|
||||
minLen: 20,
|
||||
wantCount: 1,
|
||||
wantFirst: "kR9mPx2vBnQ8jL5wYz3hTf6aU7cD",
|
||||
},
|
||||
"skips low entropy": {
|
||||
line: `password = "aaaaaaaaaaaaaaaaaaaaaa"`,
|
||||
charset: Base64Charset,
|
||||
threshold: 3.5,
|
||||
minLen: 20,
|
||||
wantCount: 0,
|
||||
},
|
||||
"skips short tokens": {
|
||||
line: `key = "abc"`,
|
||||
charset: Base64Charset,
|
||||
threshold: 1.0,
|
||||
minLen: 20,
|
||||
wantCount: 0,
|
||||
},
|
||||
"multiple tokens in one line": {
|
||||
line: `FIRST="kR9mPx2vBnQ8jL5wYz3hTf" ` +
|
||||
`SECOND="aX7bC4dE9fG2hI5jK8mN"`,
|
||||
charset: Base64Charset,
|
||||
threshold: 3.0,
|
||||
minLen: 20,
|
||||
wantCount: 2,
|
||||
},
|
||||
"no tokens in plain text": {
|
||||
line: "this is just a normal line of code",
|
||||
charset: Base64Charset,
|
||||
threshold: 4.0,
|
||||
minLen: 20,
|
||||
wantCount: 0,
|
||||
},
|
||||
}
|
||||
|
||||
for name, tc := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
tokens := ExtractHighEntropyTokens(
|
||||
tc.line,
|
||||
tc.charset,
|
||||
tc.threshold,
|
||||
tc.minLen,
|
||||
)
|
||||
require.Len(t, tokens, tc.wantCount)
|
||||
if tc.wantCount > 0 && tc.wantFirst != "" {
|
||||
assert.Equal(t, tc.wantFirst, tokens[0].Value)
|
||||
assert.Greater(
|
||||
t,
|
||||
tokens[0].Entropy,
|
||||
tc.threshold,
|
||||
)
|
||||
assert.False(
|
||||
t,
|
||||
math.IsNaN(tokens[0].Entropy),
|
||||
)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,167 @@
|
|||
// ©AngelaMos | 2026
|
||||
// registry.go
|
||||
|
||||
package rules
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/CarterPerez-dev/portia/pkg/types"
|
||||
)
|
||||
|
||||
type Registry struct {
|
||||
rules map[string]*types.Rule
|
||||
disabled map[string]bool
|
||||
}
|
||||
|
||||
func NewRegistry() *Registry {
|
||||
return &Registry{
|
||||
rules: make(map[string]*types.Rule),
|
||||
disabled: make(map[string]bool),
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Registry) Register(rule *types.Rule) {
|
||||
if _, exists := r.rules[rule.ID]; exists {
|
||||
panic(fmt.Sprintf(
|
||||
"duplicate rule ID: %s", rule.ID,
|
||||
))
|
||||
}
|
||||
r.rules[rule.ID] = rule
|
||||
}
|
||||
|
||||
func (r *Registry) Get(
|
||||
id string,
|
||||
) (*types.Rule, bool) {
|
||||
rule, ok := r.rules[id]
|
||||
if !ok || r.disabled[id] {
|
||||
return nil, false
|
||||
}
|
||||
return rule, true
|
||||
}
|
||||
|
||||
func (r *Registry) All() []*types.Rule {
|
||||
result := make([]*types.Rule, 0, len(r.rules))
|
||||
for _, rule := range r.rules {
|
||||
if !r.disabled[rule.ID] {
|
||||
result = append(result, rule)
|
||||
}
|
||||
}
|
||||
sort.Slice(result, func(i, j int) bool {
|
||||
return result[i].ID < result[j].ID
|
||||
})
|
||||
return result
|
||||
}
|
||||
|
||||
func (r *Registry) Disable(ids ...string) {
|
||||
for _, id := range ids {
|
||||
r.disabled[id] = true
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Registry) MatchKeywords(
|
||||
content string,
|
||||
) []*types.Rule {
|
||||
lower := strings.ToLower(content)
|
||||
var matched []*types.Rule
|
||||
|
||||
for _, rule := range r.rules {
|
||||
if r.disabled[rule.ID] {
|
||||
continue
|
||||
}
|
||||
for _, kw := range rule.Keywords {
|
||||
if strings.Contains(lower, strings.ToLower(kw)) {
|
||||
matched = append(matched, rule)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return matched
|
||||
}
|
||||
|
||||
func (r *Registry) Len() int {
|
||||
count := 0
|
||||
for id := range r.rules {
|
||||
if !r.disabled[id] {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
func (r *Registry) Replace(rule *types.Rule) {
|
||||
r.rules[rule.ID] = rule
|
||||
}
|
||||
|
||||
var GlobalPathAllowlist = []*regexp.Regexp{
|
||||
regexp.MustCompile(
|
||||
`go\.(?:mod|sum|work(?:\.sum)?)$`,
|
||||
),
|
||||
regexp.MustCompile(
|
||||
`(?:^|/)(?:package-lock\.json|pnpm-lock\.yaml|` +
|
||||
`yarn\.lock|npm-shrinkwrap\.json|deno\.lock|` +
|
||||
`Cargo\.lock|composer\.lock|Gemfile\.lock|` +
|
||||
`poetry\.lock|Pipfile\.lock|mix\.lock|` +
|
||||
`pubspec\.lock|Podfile\.lock|flake\.lock|` +
|
||||
`bun\.lockb)$`,
|
||||
),
|
||||
regexp.MustCompile(`(?:^|/)node_modules/`),
|
||||
regexp.MustCompile(`(?:^|/)vendor/`),
|
||||
regexp.MustCompile(`(?:^|/)\.git/`),
|
||||
regexp.MustCompile(`(?:^|/)\.svn/`),
|
||||
regexp.MustCompile(`(?:^|/)\.hg/`),
|
||||
regexp.MustCompile(
|
||||
`(?:^|/)(?:__pycache__|\.venv|venv|\.tox|` +
|
||||
`\.mypy_cache|\.pytest_cache|\.ruff_cache|` +
|
||||
`\.eggs|.*\.egg-info)/`,
|
||||
),
|
||||
regexp.MustCompile(
|
||||
`(?:^|/)(?:\.next|\.nuxt|\.svelte-kit|` +
|
||||
`\.terraform|\.gradle|\.mvn|\.bundle|` +
|
||||
`Pods|coverage|\.nyc_output)/`,
|
||||
),
|
||||
regexp.MustCompile(
|
||||
`(?:^|/)(?:target|build|dist|out)/`,
|
||||
),
|
||||
regexp.MustCompile(
|
||||
`\.min\.(?:js|css)(?:\.map)?$`,
|
||||
),
|
||||
regexp.MustCompile(
|
||||
`\.(?:png|jpg|jpeg|gif|ico|svg|woff2?|ttf|eot|otf|` +
|
||||
`mp[34]|avi|mkv|mov|zip|tar|gz|rar|7z|pdf|exe|` +
|
||||
`dll|so|dylib)$`,
|
||||
),
|
||||
regexp.MustCompile(`dist-info/`),
|
||||
}
|
||||
|
||||
var GlobalValueAllowlist = []*regexp.Regexp{
|
||||
regexp.MustCompile(
|
||||
`(?i)^(?:example|test|dummy|fake|placeholder|` +
|
||||
`sample|your[-_]?(?:api[-_]?key|token|` +
|
||||
`secret|password))`,
|
||||
),
|
||||
regexp.MustCompile(`^x{4,}$`),
|
||||
regexp.MustCompile(`^\*{4,}$`),
|
||||
regexp.MustCompile(`^\$\{.+\}$`),
|
||||
regexp.MustCompile(`^\{\{.+\}\}$`),
|
||||
regexp.MustCompile(`^<[A-Z_]{2,}>$`),
|
||||
regexp.MustCompile(`^0{8,}$`),
|
||||
regexp.MustCompile(
|
||||
`^(?:1{8,}|2{8,}|3{8,}|4{8,}|5{8,}|6{8,}|7{8,}|8{8,}|9{8,})$`,
|
||||
),
|
||||
regexp.MustCompile(
|
||||
`(?i)^(?:none|null|nil|undefined|true|false|` +
|
||||
`TODO|FIXME|CHANGEME|INSERT|REPLACE|` +
|
||||
`REDACTED|N/A|TBD|REMOVED|MASKED|` +
|
||||
`changeit|s3cr3t|p@ssw0rd|admin123)$`,
|
||||
),
|
||||
regexp.MustCompile(
|
||||
`(?i)^(?:dGVzdA==|cGFzc3dvcmQ=|YWRtaW4=)$`,
|
||||
),
|
||||
regexp.MustCompile(
|
||||
`(?i)^(?:PUT_YOUR_|ENTER_YOUR_|ADD_YOUR_)`,
|
||||
),
|
||||
}
|
||||
|
|
@ -0,0 +1,310 @@
|
|||
// ©AngelaMos | 2026
|
||||
// registry_test.go
|
||||
|
||||
package rules
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/CarterPerez-dev/portia/pkg/types"
|
||||
)
|
||||
|
||||
func testRule(id string, keywords []string) *types.Rule {
|
||||
return &types.Rule{
|
||||
ID: id,
|
||||
Keywords: keywords,
|
||||
Pattern: regexp.MustCompile(`test`),
|
||||
Severity: types.SeverityHigh,
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistryRegisterAndGet(t *testing.T) {
|
||||
t.Parallel()
|
||||
reg := NewRegistry()
|
||||
rule := testRule("test-rule", []string{"test"})
|
||||
reg.Register(rule)
|
||||
|
||||
got, ok := reg.Get("test-rule")
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, "test-rule", got.ID)
|
||||
}
|
||||
|
||||
func TestRegistryGetMissing(t *testing.T) {
|
||||
t.Parallel()
|
||||
reg := NewRegistry()
|
||||
_, ok := reg.Get("nonexistent")
|
||||
assert.False(t, ok)
|
||||
}
|
||||
|
||||
func TestRegistryDuplicatePanics(t *testing.T) {
|
||||
t.Parallel()
|
||||
reg := NewRegistry()
|
||||
rule := testRule("dup", []string{"test"})
|
||||
reg.Register(rule)
|
||||
|
||||
assert.Panics(t, func() {
|
||||
reg.Register(testRule("dup", []string{"test"}))
|
||||
})
|
||||
}
|
||||
|
||||
func TestRegistryAll(t *testing.T) {
|
||||
t.Parallel()
|
||||
reg := NewRegistry()
|
||||
reg.Register(testRule("b-rule", []string{"b"}))
|
||||
reg.Register(testRule("a-rule", []string{"a"}))
|
||||
reg.Register(testRule("c-rule", []string{"c"}))
|
||||
|
||||
all := reg.All()
|
||||
require.Len(t, all, 3)
|
||||
assert.Equal(t, "a-rule", all[0].ID)
|
||||
assert.Equal(t, "b-rule", all[1].ID)
|
||||
assert.Equal(t, "c-rule", all[2].ID)
|
||||
}
|
||||
|
||||
func TestRegistryDisable(t *testing.T) {
|
||||
t.Parallel()
|
||||
reg := NewRegistry()
|
||||
reg.Register(testRule("keep", []string{"a"}))
|
||||
reg.Register(testRule("drop", []string{"b"}))
|
||||
|
||||
reg.Disable("drop")
|
||||
|
||||
_, ok := reg.Get("drop")
|
||||
assert.False(t, ok)
|
||||
|
||||
all := reg.All()
|
||||
require.Len(t, all, 1)
|
||||
assert.Equal(t, "keep", all[0].ID)
|
||||
}
|
||||
|
||||
func TestRegistryMatchKeywords(t *testing.T) {
|
||||
t.Parallel()
|
||||
reg := NewRegistry()
|
||||
reg.Register(
|
||||
testRule("aws", []string{"AKIA", "aws"}),
|
||||
)
|
||||
reg.Register(
|
||||
testRule("github", []string{"ghp_"}),
|
||||
)
|
||||
reg.Register(
|
||||
testRule("stripe", []string{"sk_live_"}),
|
||||
)
|
||||
|
||||
tests := map[string]struct {
|
||||
content string
|
||||
wantIDs []string
|
||||
wantNone bool
|
||||
}{
|
||||
"matches aws by AKIA prefix": { //nolint:gosec
|
||||
content: "found AKIAIOSFODNN7EXAMPLE in config",
|
||||
wantIDs: []string{"aws"},
|
||||
},
|
||||
"matches github token": {
|
||||
content: "token = ghp_abc123def456",
|
||||
wantIDs: []string{"github"},
|
||||
},
|
||||
"matches multiple rules": {
|
||||
content: "aws key AKIA and ghp_ token here",
|
||||
wantIDs: []string{"aws", "github"},
|
||||
},
|
||||
"case insensitive matching": {
|
||||
content: "AWS_ACCESS_KEY with akia prefix",
|
||||
wantIDs: []string{"aws"},
|
||||
},
|
||||
"no matches on unrelated content": {
|
||||
content: "just some normal code here",
|
||||
wantNone: true,
|
||||
},
|
||||
}
|
||||
|
||||
for name, tc := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
matched := reg.MatchKeywords(tc.content)
|
||||
if tc.wantNone {
|
||||
assert.Empty(t, matched)
|
||||
return
|
||||
}
|
||||
ids := make([]string, len(matched))
|
||||
for i, r := range matched {
|
||||
ids[i] = r.ID
|
||||
}
|
||||
for _, wantID := range tc.wantIDs {
|
||||
assert.Contains(t, ids, wantID)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistryMatchKeywordsSkipsDisabled(t *testing.T) {
|
||||
t.Parallel()
|
||||
reg := NewRegistry()
|
||||
reg.Register(
|
||||
testRule("enabled", []string{"secret"}),
|
||||
)
|
||||
reg.Register(
|
||||
testRule("disabled", []string{"secret"}),
|
||||
)
|
||||
reg.Disable("disabled")
|
||||
|
||||
matched := reg.MatchKeywords("my secret value")
|
||||
require.Len(t, matched, 1)
|
||||
assert.Equal(t, "enabled", matched[0].ID)
|
||||
}
|
||||
|
||||
func TestRegistryLen(t *testing.T) {
|
||||
t.Parallel()
|
||||
reg := NewRegistry()
|
||||
reg.Register(testRule("a", []string{"a"}))
|
||||
reg.Register(testRule("b", []string{"b"}))
|
||||
assert.Equal(t, 2, reg.Len())
|
||||
|
||||
reg.Disable("a")
|
||||
assert.Equal(t, 1, reg.Len())
|
||||
}
|
||||
|
||||
func TestRegistryReplace(t *testing.T) {
|
||||
t.Parallel()
|
||||
reg := NewRegistry()
|
||||
reg.Register(testRule("r", []string{"old"}))
|
||||
|
||||
newRule := testRule("r", []string{"new"})
|
||||
newRule.Description = "replaced"
|
||||
reg.Replace(newRule)
|
||||
|
||||
got, ok := reg.Get("r")
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, "replaced", got.Description)
|
||||
}
|
||||
|
||||
func TestGlobalPathAllowlist(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
allowed := []string{
|
||||
"go.sum",
|
||||
"go.mod",
|
||||
"package-lock.json",
|
||||
"pnpm-lock.yaml",
|
||||
"node_modules/pkg/index.js",
|
||||
"vendor/lib/code.go",
|
||||
".git/objects/pack/abc",
|
||||
"dist/bundle.min.js",
|
||||
"assets/logo.png",
|
||||
"data/file.pdf",
|
||||
"Cargo.lock",
|
||||
"composer.lock",
|
||||
"Gemfile.lock",
|
||||
"poetry.lock",
|
||||
"Pipfile.lock",
|
||||
"bun.lockb",
|
||||
".svn/entries",
|
||||
".hg/store/data",
|
||||
"__pycache__/module.cpython-313.pyc",
|
||||
".mypy_cache/3.13/module.meta.json",
|
||||
".next/server/pages/index.html",
|
||||
".terraform/providers/registry.terraform.io",
|
||||
"target/debug/build/crate/output",
|
||||
"dist/bundle.min.css",
|
||||
"dist/bundle.min.css.map",
|
||||
"assets/font.otf",
|
||||
}
|
||||
|
||||
blocked := []string{
|
||||
"src/main.go",
|
||||
"config.yaml",
|
||||
".env",
|
||||
"internal/auth/handler.go",
|
||||
}
|
||||
|
||||
for _, path := range allowed {
|
||||
matched := false
|
||||
for _, re := range GlobalPathAllowlist {
|
||||
if re.MatchString(path) {
|
||||
matched = true
|
||||
break
|
||||
}
|
||||
}
|
||||
assert.True(t, matched,
|
||||
"expected %s to be allowlisted", path)
|
||||
}
|
||||
|
||||
for _, path := range blocked {
|
||||
matched := false
|
||||
for _, re := range GlobalPathAllowlist {
|
||||
if re.MatchString(path) {
|
||||
matched = true
|
||||
break
|
||||
}
|
||||
}
|
||||
assert.False(t, matched,
|
||||
"expected %s to NOT be allowlisted", path)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGlobalValueAllowlist(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
allowed := []string{
|
||||
"EXAMPLE_KEY",
|
||||
"test_token",
|
||||
"your-api-key",
|
||||
"placeholder",
|
||||
"xxxxxxxxxxxx",
|
||||
"****",
|
||||
"${SECRET_KEY}",
|
||||
"{{api_token}}",
|
||||
"CHANGEME",
|
||||
"none",
|
||||
"undefined",
|
||||
"<YOUR_API_KEY>",
|
||||
"<REPLACE_ME>",
|
||||
"<INSERT_TOKEN>",
|
||||
"00000000000000000000",
|
||||
"1111111111111111",
|
||||
"REDACTED",
|
||||
"N/A",
|
||||
"TBD",
|
||||
"REMOVED",
|
||||
"MASKED",
|
||||
"changeit",
|
||||
"PUT_YOUR_KEY_HERE",
|
||||
"ENTER_YOUR_TOKEN_HERE",
|
||||
"dGVzdA==",
|
||||
"cGFzc3dvcmQ=",
|
||||
}
|
||||
|
||||
blocked := []string{
|
||||
"AKIAIOSFODNN7EXAMPLE",
|
||||
"sk_live_4eC39HqLyjWDarjtT1",
|
||||
"ghp_ABCDEFGHIJKLMNOPQRSTUVWXYZab",
|
||||
"xK9mP2vL5nQ8",
|
||||
}
|
||||
|
||||
for _, val := range allowed {
|
||||
matched := false
|
||||
for _, re := range GlobalValueAllowlist {
|
||||
if re.MatchString(val) {
|
||||
matched = true
|
||||
break
|
||||
}
|
||||
}
|
||||
assert.True(t, matched,
|
||||
"expected %q to be allowlisted", val)
|
||||
}
|
||||
|
||||
for _, val := range blocked {
|
||||
matched := false
|
||||
for _, re := range GlobalValueAllowlist {
|
||||
if re.MatchString(val) {
|
||||
matched = true
|
||||
break
|
||||
}
|
||||
}
|
||||
assert.False(t, matched,
|
||||
"expected %q to NOT be allowlisted", val)
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,209 @@
|
|||
// ©AngelaMos | 2026
|
||||
// directory.go
|
||||
|
||||
package source
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/CarterPerez-dev/portia/pkg/types"
|
||||
)
|
||||
|
||||
const defaultMaxFileSize = 1 << 20
|
||||
|
||||
type Directory struct {
|
||||
Path string
|
||||
MaxSize int64
|
||||
Excludes []string
|
||||
}
|
||||
|
||||
func NewDirectory(
|
||||
path string, maxSize int64, excludes []string,
|
||||
) *Directory {
|
||||
if maxSize <= 0 {
|
||||
maxSize = defaultMaxFileSize
|
||||
}
|
||||
return &Directory{
|
||||
Path: path,
|
||||
MaxSize: maxSize,
|
||||
Excludes: excludes,
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Directory) String() string {
|
||||
return "directory:" + d.Path
|
||||
}
|
||||
|
||||
func (d *Directory) Chunks(
|
||||
ctx context.Context, out chan<- types.Chunk,
|
||||
) error {
|
||||
return filepath.WalkDir(
|
||||
d.Path,
|
||||
func(path string, entry fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return nil //nolint:nilerr
|
||||
}
|
||||
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
if entry.IsDir() {
|
||||
base := entry.Name()
|
||||
if base == ".git" || base == "node_modules" ||
|
||||
base == "vendor" || base == "__pycache__" ||
|
||||
base == ".venv" || base == "venv" ||
|
||||
base == ".svn" || base == ".hg" ||
|
||||
base == ".tox" || base == ".mypy_cache" ||
|
||||
base == ".pytest_cache" || base == ".ruff_cache" ||
|
||||
base == ".next" || base == ".nuxt" ||
|
||||
base == ".terraform" || base == ".gradle" ||
|
||||
base == "Pods" || base == "coverage" ||
|
||||
base == ".nyc_output" || base == ".bundle" ||
|
||||
base == "target" || base == ".eggs" {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
rel, relErr := filepath.Rel(d.Path, path)
|
||||
if relErr != nil {
|
||||
rel = path
|
||||
}
|
||||
|
||||
if d.isExcluded(rel) {
|
||||
return nil
|
||||
}
|
||||
|
||||
if isBinaryExt(path) {
|
||||
return nil
|
||||
}
|
||||
|
||||
info, infoErr := entry.Info()
|
||||
if infoErr != nil || info.Size() > d.MaxSize {
|
||||
return nil //nolint:nilerr
|
||||
}
|
||||
|
||||
return d.emitChunks(ctx, path, rel, out)
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
func (d *Directory) emitChunks(
|
||||
ctx context.Context,
|
||||
absPath, relPath string,
|
||||
out chan<- types.Chunk,
|
||||
) error {
|
||||
f, err := os.Open(absPath) //nolint:gosec
|
||||
if err != nil {
|
||||
return nil //nolint:nilerr
|
||||
}
|
||||
defer f.Close() //nolint:errcheck
|
||||
|
||||
var buf strings.Builder
|
||||
scanner := bufio.NewScanner(f)
|
||||
scanner.Buffer(make([]byte, 0, 512*1024), 512*1024)
|
||||
|
||||
lineNum := 0
|
||||
chunkStart := 1
|
||||
linesInChunk := 0
|
||||
|
||||
for scanner.Scan() {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
lineNum++
|
||||
linesInChunk++
|
||||
|
||||
if buf.Len() > 0 {
|
||||
buf.WriteByte('\n')
|
||||
}
|
||||
buf.WriteString(scanner.Text())
|
||||
|
||||
if linesInChunk >= 50 {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case out <- types.Chunk{
|
||||
Content: buf.String(),
|
||||
FilePath: relPath,
|
||||
LineStart: chunkStart,
|
||||
}:
|
||||
}
|
||||
buf.Reset()
|
||||
chunkStart = lineNum + 1
|
||||
linesInChunk = 0
|
||||
}
|
||||
}
|
||||
|
||||
if buf.Len() > 0 {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case out <- types.Chunk{
|
||||
Content: buf.String(),
|
||||
FilePath: relPath,
|
||||
LineStart: chunkStart,
|
||||
}:
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Directory) isExcluded(relPath string) bool {
|
||||
for _, pattern := range d.Excludes {
|
||||
if matched, _ := filepath.Match( //nolint:errcheck
|
||||
pattern, filepath.Base(relPath),
|
||||
); matched {
|
||||
return true
|
||||
}
|
||||
if strings.Contains(relPath, pattern) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
var binaryExts = map[string]bool{
|
||||
".png": true, ".jpg": true, ".jpeg": true,
|
||||
".gif": true, ".ico": true, ".svg": true,
|
||||
".bmp": true, ".tiff": true, ".tif": true,
|
||||
".webp": true, ".avif": true, ".heic": true,
|
||||
".heif": true, ".psd": true, ".raw": true,
|
||||
".cr2": true,
|
||||
".woff": true, ".woff2": true, ".ttf": true,
|
||||
".eot": true, ".otf": true,
|
||||
".mp3": true, ".mp4": true, ".avi": true,
|
||||
".mkv": true, ".mov": true, ".wmv": true,
|
||||
".flv": true, ".webm": true, ".flac": true,
|
||||
".wav": true, ".ogg": true, ".aac": true,
|
||||
".m4a": true,
|
||||
".zip": true, ".tar": true, ".gz": true,
|
||||
".rar": true, ".7z": true, ".bz2": true,
|
||||
".xz": true, ".zst": true, ".lz4": true,
|
||||
".lzma": true, ".cab": true,
|
||||
".deb": true, ".rpm": true, ".msi": true,
|
||||
".dmg": true, ".iso": true, ".img": true,
|
||||
".snap": true,
|
||||
".pdf": true, ".exe": true, ".dll": true,
|
||||
".so": true, ".dylib": true, ".bin": true,
|
||||
".o": true, ".a": true, ".class": true,
|
||||
".jar": true, ".war": true, ".ear": true,
|
||||
".pyc": true, ".pyo": true, ".wasm": true,
|
||||
".db": true, ".sqlite": true, ".sqlite3": true,
|
||||
".dat": true, ".npy": true, ".npz": true,
|
||||
".h5": true, ".hdf5": true, ".parquet": true,
|
||||
".tfrecord": true, ".pkl": true, ".pickle": true,
|
||||
".pt": true, ".onnx": true, ".safetensors": true,
|
||||
}
|
||||
|
||||
func isBinaryExt(path string) bool {
|
||||
return binaryExts[strings.ToLower(filepath.Ext(path))]
|
||||
}
|
||||
|
|
@ -0,0 +1,313 @@
|
|||
// ©AngelaMos | 2026
|
||||
// git.go
|
||||
|
||||
package source
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/go-git/go-git/v5"
|
||||
"github.com/go-git/go-git/v5/plumbing"
|
||||
"github.com/go-git/go-git/v5/plumbing/object"
|
||||
"github.com/go-git/go-git/v5/plumbing/storer"
|
||||
|
||||
"github.com/CarterPerez-dev/portia/pkg/types"
|
||||
)
|
||||
|
||||
type Git struct {
|
||||
RepoPath string
|
||||
Branch string
|
||||
Since string
|
||||
Depth int
|
||||
StagedOnly bool
|
||||
MaxSize int64
|
||||
Excludes []string
|
||||
}
|
||||
|
||||
func NewGit(
|
||||
repoPath string,
|
||||
branch string,
|
||||
since string,
|
||||
depth int,
|
||||
stagedOnly bool,
|
||||
maxSize int64,
|
||||
excludes []string,
|
||||
) *Git {
|
||||
if maxSize <= 0 {
|
||||
maxSize = defaultMaxFileSize
|
||||
}
|
||||
return &Git{
|
||||
RepoPath: repoPath,
|
||||
Branch: branch,
|
||||
Since: since,
|
||||
Depth: depth,
|
||||
StagedOnly: stagedOnly,
|
||||
MaxSize: maxSize,
|
||||
Excludes: excludes,
|
||||
}
|
||||
}
|
||||
|
||||
func (g *Git) String() string {
|
||||
return "git:" + g.RepoPath
|
||||
}
|
||||
|
||||
func (g *Git) Chunks(
|
||||
ctx context.Context, out chan<- types.Chunk,
|
||||
) error {
|
||||
repo, err := git.PlainOpen(g.RepoPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open repository: %w", err)
|
||||
}
|
||||
|
||||
if g.StagedOnly {
|
||||
return g.scanStaged(ctx, repo, out)
|
||||
}
|
||||
|
||||
return g.scanHistory(ctx, repo, out)
|
||||
}
|
||||
|
||||
func (g *Git) scanStaged(
|
||||
ctx context.Context,
|
||||
repo *git.Repository,
|
||||
out chan<- types.Chunk,
|
||||
) error {
|
||||
wt, err := repo.Worktree()
|
||||
if err != nil {
|
||||
return fmt.Errorf("worktree: %w", err)
|
||||
}
|
||||
|
||||
status, err := wt.Status()
|
||||
if err != nil {
|
||||
return fmt.Errorf("status: %w", err)
|
||||
}
|
||||
|
||||
idx, err := repo.Storer.Index()
|
||||
if err != nil {
|
||||
return fmt.Errorf("index: %w", err)
|
||||
}
|
||||
|
||||
for _, entry := range idx.Entries {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
fileStatus := status.File(entry.Name)
|
||||
if fileStatus.Staging == git.Unmodified &&
|
||||
fileStatus.Worktree == git.Unmodified {
|
||||
continue
|
||||
}
|
||||
|
||||
if g.isExcluded(entry.Name) || isBinaryExt(entry.Name) {
|
||||
continue
|
||||
}
|
||||
|
||||
blob, blobErr := repo.BlobObject(entry.Hash)
|
||||
if blobErr != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
if blob.Size > g.MaxSize {
|
||||
continue
|
||||
}
|
||||
|
||||
content, readErr := readBlob(blob)
|
||||
if readErr != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
chunks := splitIntoChunks(
|
||||
content, entry.Name, "", "", time.Time{},
|
||||
)
|
||||
for _, chunk := range chunks {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case out <- chunk:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (g *Git) scanHistory( //nolint:gocognit
|
||||
ctx context.Context,
|
||||
repo *git.Repository,
|
||||
out chan<- types.Chunk,
|
||||
) error {
|
||||
ref, err := g.resolveRef(repo)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
logOpts := &git.LogOptions{
|
||||
From: ref.Hash(),
|
||||
Order: git.LogOrderCommitterTime,
|
||||
}
|
||||
|
||||
if g.Since != "" {
|
||||
sinceTime, parseErr := time.Parse("2006-01-02", g.Since)
|
||||
if parseErr == nil {
|
||||
logOpts.Since = &sinceTime
|
||||
}
|
||||
}
|
||||
|
||||
iter, err := repo.Log(logOpts)
|
||||
if err != nil {
|
||||
return fmt.Errorf("git log: %w", err)
|
||||
}
|
||||
defer iter.Close()
|
||||
|
||||
count := 0
|
||||
return iter.ForEach(func(c *object.Commit) error {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
if g.Depth > 0 && count >= g.Depth {
|
||||
return storer.ErrStop
|
||||
}
|
||||
count++
|
||||
|
||||
tree, treeErr := c.Tree()
|
||||
if treeErr != nil {
|
||||
return nil //nolint:nilerr
|
||||
}
|
||||
|
||||
return tree.Files().ForEach(func(f *object.File) error {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
if g.isExcluded(f.Name) || isBinaryExt(f.Name) {
|
||||
return nil
|
||||
}
|
||||
|
||||
if f.Size > g.MaxSize {
|
||||
return nil
|
||||
}
|
||||
|
||||
isBin, binErr := f.IsBinary()
|
||||
if binErr != nil || isBin {
|
||||
return nil //nolint:nilerr
|
||||
}
|
||||
|
||||
content, readErr := f.Contents()
|
||||
if readErr != nil {
|
||||
return nil //nolint:nilerr
|
||||
}
|
||||
|
||||
author := ""
|
||||
if c.Author.Email != "" {
|
||||
author = c.Author.Email
|
||||
}
|
||||
|
||||
chunks := splitIntoChunks(
|
||||
content,
|
||||
f.Name,
|
||||
c.Hash.String(),
|
||||
author,
|
||||
c.Author.When,
|
||||
)
|
||||
for _, chunk := range chunks {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case out <- chunk:
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func (g *Git) resolveRef(
|
||||
repo *git.Repository,
|
||||
) (*plumbing.Reference, error) {
|
||||
if g.Branch != "" {
|
||||
ref, err := repo.Reference(
|
||||
plumbing.NewBranchReferenceName(g.Branch),
|
||||
true,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf(
|
||||
"resolve branch %s: %w", g.Branch, err,
|
||||
)
|
||||
}
|
||||
return ref, nil
|
||||
}
|
||||
|
||||
ref, err := repo.Head()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("resolve HEAD: %w", err)
|
||||
}
|
||||
return ref, nil
|
||||
}
|
||||
|
||||
func (g *Git) isExcluded(path string) bool {
|
||||
for _, pattern := range g.Excludes {
|
||||
if matched, _ := filepath.Match( //nolint:errcheck
|
||||
pattern, filepath.Base(path),
|
||||
); matched {
|
||||
return true
|
||||
}
|
||||
if strings.Contains(path, pattern) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func readBlob(blob *object.Blob) (string, error) {
|
||||
reader, err := blob.Reader()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer reader.Close() //nolint:errcheck
|
||||
|
||||
data, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(data), nil
|
||||
}
|
||||
|
||||
func splitIntoChunks(
|
||||
content string,
|
||||
filePath string,
|
||||
commitSHA string,
|
||||
author string,
|
||||
commitDate time.Time,
|
||||
) []types.Chunk {
|
||||
lines := strings.Split(content, "\n")
|
||||
var chunks []types.Chunk
|
||||
|
||||
for i := 0; i < len(lines); i += 50 {
|
||||
end := i + 50
|
||||
if end > len(lines) {
|
||||
end = len(lines)
|
||||
}
|
||||
|
||||
chunkContent := strings.Join(lines[i:end], "\n")
|
||||
if strings.TrimSpace(chunkContent) == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
chunks = append(chunks, types.Chunk{
|
||||
Content: chunkContent,
|
||||
FilePath: filePath,
|
||||
LineStart: i + 1,
|
||||
CommitSHA: commitSHA,
|
||||
Author: author,
|
||||
CommitDate: commitDate,
|
||||
})
|
||||
}
|
||||
|
||||
return chunks
|
||||
}
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
// ©AngelaMos | 2026
|
||||
// source.go
|
||||
|
||||
package source
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/CarterPerez-dev/portia/pkg/types"
|
||||
)
|
||||
|
||||
type Source interface {
|
||||
Chunks(ctx context.Context, out chan<- types.Chunk) error
|
||||
String() string
|
||||
}
|
||||
|
|
@ -0,0 +1,409 @@
|
|||
// ©AngelaMos | 2026
|
||||
// source_test.go
|
||||
|
||||
package source
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/go-git/go-git/v5"
|
||||
"github.com/go-git/go-git/v5/plumbing/object"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/CarterPerez-dev/portia/pkg/types"
|
||||
)
|
||||
|
||||
func collectChunks(
|
||||
t *testing.T, src Source,
|
||||
) []types.Chunk {
|
||||
t.Helper()
|
||||
out := make(chan types.Chunk, 100)
|
||||
ctx := context.Background()
|
||||
|
||||
errCh := make(chan error, 1)
|
||||
go func() {
|
||||
errCh <- src.Chunks(ctx, out)
|
||||
close(out)
|
||||
}()
|
||||
|
||||
var chunks []types.Chunk
|
||||
for c := range out {
|
||||
chunks = append(chunks, c)
|
||||
}
|
||||
require.NoError(t, <-errCh)
|
||||
return chunks
|
||||
}
|
||||
|
||||
func TestDirectoryChunks(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dir := t.TempDir()
|
||||
require.NoError(t, os.WriteFile( //nolint:gosec
|
||||
filepath.Join(dir, "config.py"),
|
||||
[]byte(`password = "sk_live_4eC39HqLyjWDarjtT1zdp7dc"`+"\n"),
|
||||
0o644,
|
||||
))
|
||||
require.NoError(t, os.WriteFile( //nolint:gosec
|
||||
filepath.Join(dir, "logo.png"),
|
||||
[]byte{0x89, 0x50, 0x4E, 0x47},
|
||||
0o644,
|
||||
))
|
||||
|
||||
src := NewDirectory(dir, 0, nil)
|
||||
chunks := collectChunks(t, src)
|
||||
|
||||
require.Len(t, chunks, 1)
|
||||
assert.Equal(t, "config.py", chunks[0].FilePath)
|
||||
assert.Contains(t, chunks[0].Content, "sk_live_")
|
||||
assert.Equal(t, 1, chunks[0].LineStart)
|
||||
}
|
||||
|
||||
func TestDirectoryExcludes(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dir := t.TempDir()
|
||||
require.NoError(t, os.WriteFile( //nolint:gosec
|
||||
filepath.Join(dir, "main.go"),
|
||||
[]byte("package main\n"),
|
||||
0o644,
|
||||
))
|
||||
require.NoError(t, os.WriteFile( //nolint:gosec
|
||||
filepath.Join(dir, "secret.env"),
|
||||
[]byte("API_KEY=test\n"),
|
||||
0o644,
|
||||
))
|
||||
|
||||
src := NewDirectory(dir, 0, []string{"*.env"})
|
||||
chunks := collectChunks(t, src)
|
||||
|
||||
require.Len(t, chunks, 1)
|
||||
assert.Equal(t, "main.go", chunks[0].FilePath)
|
||||
}
|
||||
|
||||
func TestDirectorySkipsDotGit(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dir := t.TempDir()
|
||||
gitDir := filepath.Join(dir, ".git", "objects")
|
||||
require.NoError(t, os.MkdirAll(gitDir, 0o755)) //nolint:gosec
|
||||
require.NoError(t, os.WriteFile( //nolint:gosec
|
||||
filepath.Join(gitDir, "data"),
|
||||
[]byte("git internal data\n"),
|
||||
0o644,
|
||||
))
|
||||
require.NoError(t, os.WriteFile( //nolint:gosec
|
||||
filepath.Join(dir, "app.go"),
|
||||
[]byte("package main\n"),
|
||||
0o644,
|
||||
))
|
||||
|
||||
src := NewDirectory(dir, 0, nil)
|
||||
chunks := collectChunks(t, src)
|
||||
|
||||
require.Len(t, chunks, 1)
|
||||
assert.Equal(t, "app.go", chunks[0].FilePath)
|
||||
}
|
||||
|
||||
func TestDirectoryMaxSize(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dir := t.TempDir()
|
||||
require.NoError(t, os.WriteFile( //nolint:gosec
|
||||
filepath.Join(dir, "small.txt"),
|
||||
[]byte("small\n"),
|
||||
0o644,
|
||||
))
|
||||
bigContent := make([]byte, 2*1024*1024)
|
||||
for i := range bigContent {
|
||||
bigContent[i] = 'x'
|
||||
}
|
||||
require.NoError(t, os.WriteFile( //nolint:gosec
|
||||
filepath.Join(dir, "big.txt"),
|
||||
bigContent,
|
||||
0o644,
|
||||
))
|
||||
|
||||
src := NewDirectory(dir, 1024*1024, nil)
|
||||
chunks := collectChunks(t, src)
|
||||
|
||||
require.Len(t, chunks, 1)
|
||||
assert.Equal(t, "small.txt", chunks[0].FilePath)
|
||||
}
|
||||
|
||||
func TestDirectoryChunking(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dir := t.TempDir()
|
||||
var lines string
|
||||
for i := range 120 {
|
||||
lines += "line " + string(rune('A'+i%26)) + "\n"
|
||||
}
|
||||
require.NoError(t, os.WriteFile( //nolint:gosec
|
||||
filepath.Join(dir, "long.txt"),
|
||||
[]byte(lines),
|
||||
0o644,
|
||||
))
|
||||
|
||||
src := NewDirectory(dir, 0, nil)
|
||||
chunks := collectChunks(t, src)
|
||||
|
||||
require.GreaterOrEqual(t, len(chunks), 2)
|
||||
assert.Equal(t, 1, chunks[0].LineStart)
|
||||
assert.Equal(t, 51, chunks[1].LineStart)
|
||||
}
|
||||
|
||||
func TestDirectoryContextCancel(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dir := t.TempDir()
|
||||
for i := range 10 {
|
||||
name := filepath.Join(dir, "file"+string(rune('0'+i))+".txt")
|
||||
require.NoError(t, os.WriteFile( //nolint:gosec
|
||||
name, []byte("content\n"), 0o644,
|
||||
))
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
src := NewDirectory(dir, 0, nil)
|
||||
out := make(chan types.Chunk, 100)
|
||||
err := src.Chunks(ctx, out)
|
||||
close(out)
|
||||
|
||||
assert.ErrorIs(t, err, context.Canceled)
|
||||
}
|
||||
|
||||
func TestDirectoryString(t *testing.T) {
|
||||
t.Parallel()
|
||||
src := NewDirectory("/tmp/test", 0, nil)
|
||||
assert.Equal(t, "directory:/tmp/test", src.String())
|
||||
}
|
||||
|
||||
func TestGitChunks(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dir := t.TempDir()
|
||||
repo, err := git.PlainInit(dir, false)
|
||||
require.NoError(t, err)
|
||||
|
||||
wt, err := repo.Worktree()
|
||||
require.NoError(t, err)
|
||||
|
||||
require.NoError(t, os.WriteFile( //nolint:gosec
|
||||
filepath.Join(dir, "secret.py"),
|
||||
[]byte(`api_key = "sk_live_4eC39HqLyjWDarjtT1zdp7dc"`+"\n"),
|
||||
0o644,
|
||||
))
|
||||
_, err = wt.Add("secret.py")
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = wt.Commit("add secret", &git.CommitOptions{
|
||||
Author: &object.Signature{
|
||||
Name: "Test",
|
||||
Email: "test@example.com",
|
||||
When: time.Now(),
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
src := NewGit(dir, "", "", 0, false, 0, nil)
|
||||
chunks := collectChunks(t, src)
|
||||
|
||||
require.GreaterOrEqual(t, len(chunks), 1)
|
||||
assert.Equal(t, "secret.py", chunks[0].FilePath)
|
||||
assert.Contains(t, chunks[0].Content, "sk_live_")
|
||||
assert.NotEmpty(t, chunks[0].CommitSHA)
|
||||
assert.Equal(t, "test@example.com", chunks[0].Author)
|
||||
}
|
||||
|
||||
func TestGitStagedOnly(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dir := t.TempDir()
|
||||
repo, err := git.PlainInit(dir, false)
|
||||
require.NoError(t, err)
|
||||
|
||||
wt, err := repo.Worktree()
|
||||
require.NoError(t, err)
|
||||
|
||||
require.NoError(t, os.WriteFile( //nolint:gosec
|
||||
filepath.Join(dir, "committed.txt"),
|
||||
[]byte("committed content\n"),
|
||||
0o644,
|
||||
))
|
||||
_, err = wt.Add("committed.txt")
|
||||
require.NoError(t, err)
|
||||
_, err = wt.Commit("initial", &git.CommitOptions{
|
||||
Author: &object.Signature{
|
||||
Name: "Test",
|
||||
Email: "test@example.com",
|
||||
When: time.Now(),
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
require.NoError(t, os.WriteFile( //nolint:gosec
|
||||
filepath.Join(dir, "staged.txt"),
|
||||
[]byte("new staged content\n"),
|
||||
0o644,
|
||||
))
|
||||
_, err = wt.Add("staged.txt")
|
||||
require.NoError(t, err)
|
||||
|
||||
src := NewGit(dir, "", "", 0, true, 0, nil)
|
||||
chunks := collectChunks(t, src)
|
||||
|
||||
found := false
|
||||
for _, c := range chunks {
|
||||
if c.FilePath == "staged.txt" {
|
||||
found = true
|
||||
assert.Contains(t, c.Content, "new staged content")
|
||||
}
|
||||
}
|
||||
assert.True(t, found, "expected to find staged.txt chunk")
|
||||
}
|
||||
|
||||
func TestGitDepthLimit(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dir := t.TempDir()
|
||||
repo, err := git.PlainInit(dir, false)
|
||||
require.NoError(t, err)
|
||||
|
||||
wt, err := repo.Worktree()
|
||||
require.NoError(t, err)
|
||||
|
||||
for i := range 5 {
|
||||
name := "file" + string(rune('0'+i)) + ".txt"
|
||||
require.NoError(t, os.WriteFile( //nolint:gosec
|
||||
filepath.Join(dir, name),
|
||||
[]byte("content "+name+"\n"),
|
||||
0o644,
|
||||
))
|
||||
_, err = wt.Add(name)
|
||||
require.NoError(t, err)
|
||||
_, err = wt.Commit("commit "+name, &git.CommitOptions{
|
||||
Author: &object.Signature{
|
||||
Name: "Test",
|
||||
Email: "test@example.com",
|
||||
When: time.Now().Add(time.Duration(i) * time.Minute),
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
src := NewGit(dir, "", "", 2, false, 0, nil)
|
||||
chunks := collectChunks(t, src)
|
||||
|
||||
files := make(map[string]bool)
|
||||
for _, c := range chunks {
|
||||
files[c.FilePath] = true
|
||||
}
|
||||
assert.LessOrEqual(t, len(files), 5)
|
||||
}
|
||||
|
||||
func TestGitExcludes(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dir := t.TempDir()
|
||||
repo, err := git.PlainInit(dir, false)
|
||||
require.NoError(t, err)
|
||||
|
||||
wt, err := repo.Worktree()
|
||||
require.NoError(t, err)
|
||||
|
||||
require.NoError(t, os.WriteFile( //nolint:gosec
|
||||
filepath.Join(dir, "keep.go"),
|
||||
[]byte("package main\n"),
|
||||
0o644,
|
||||
))
|
||||
require.NoError(t, os.WriteFile( //nolint:gosec
|
||||
filepath.Join(dir, "skip.lock"),
|
||||
[]byte("lock data\n"),
|
||||
0o644,
|
||||
))
|
||||
_, err = wt.Add("keep.go")
|
||||
require.NoError(t, err)
|
||||
_, err = wt.Add("skip.lock")
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = wt.Commit("add files", &git.CommitOptions{
|
||||
Author: &object.Signature{
|
||||
Name: "Test",
|
||||
Email: "test@example.com",
|
||||
When: time.Now(),
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
src := NewGit(dir, "", "", 0, false, 0, []string{".lock"})
|
||||
chunks := collectChunks(t, src)
|
||||
|
||||
for _, c := range chunks {
|
||||
assert.NotContains(t, c.FilePath, ".lock")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGitString(t *testing.T) {
|
||||
t.Parallel()
|
||||
src := NewGit("/tmp/repo", "", "", 0, false, 0, nil)
|
||||
assert.Equal(t, "git:/tmp/repo", src.String())
|
||||
}
|
||||
|
||||
func TestSplitIntoChunks(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var lines []string
|
||||
for i := range 120 {
|
||||
lines = append(lines, "line "+string(rune('A'+i%26)))
|
||||
}
|
||||
content := ""
|
||||
for i, l := range lines {
|
||||
if i > 0 {
|
||||
content += "\n"
|
||||
}
|
||||
content += l
|
||||
}
|
||||
|
||||
chunks := splitIntoChunks(
|
||||
content, "test.txt", "abc123", "dev@test.com", time.Now(),
|
||||
)
|
||||
|
||||
require.Len(t, chunks, 3)
|
||||
assert.Equal(t, 1, chunks[0].LineStart)
|
||||
assert.Equal(t, 51, chunks[1].LineStart)
|
||||
assert.Equal(t, 101, chunks[2].LineStart)
|
||||
assert.Equal(t, "abc123", chunks[0].CommitSHA)
|
||||
assert.Equal(t, "dev@test.com", chunks[0].Author)
|
||||
}
|
||||
|
||||
func TestSplitIntoChunksEmpty(t *testing.T) {
|
||||
t.Parallel()
|
||||
chunks := splitIntoChunks(
|
||||
"", "test.txt", "", "", time.Time{},
|
||||
)
|
||||
assert.Empty(t, chunks)
|
||||
}
|
||||
|
||||
func TestIsBinaryExt(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
assert.True(t, isBinaryExt("image.png"))
|
||||
assert.True(t, isBinaryExt("data.PDF"))
|
||||
assert.True(t, isBinaryExt("app.exe"))
|
||||
assert.True(t, isBinaryExt("model.onnx"))
|
||||
assert.True(t, isBinaryExt("font.otf"))
|
||||
assert.True(t, isBinaryExt("data.parquet"))
|
||||
assert.True(t, isBinaryExt("archive.xz"))
|
||||
assert.True(t, isBinaryExt("package.deb"))
|
||||
assert.True(t, isBinaryExt("movie.mkv"))
|
||||
assert.False(t, isBinaryExt("main.go"))
|
||||
assert.False(t, isBinaryExt("config.yaml"))
|
||||
assert.False(t, isBinaryExt("script.py"))
|
||||
}
|
||||
|
|
@ -0,0 +1,85 @@
|
|||
// ©AngelaMos | 2026
|
||||
// banner.go
|
||||
|
||||
package ui
|
||||
|
||||
import "fmt"
|
||||
|
||||
var portiaBanner = []string{
|
||||
" ",
|
||||
"░░░░░░ ░░░░░░ ░░░░░░ ░░░░░░░░ ░░ ░░░░░ ",
|
||||
"▒▒ ▒▒ ▒▒ ▒▒ ▒▒ ▒▒ ▒▒ ▒▒ ▒▒ ▒▒ ",
|
||||
"▒▒▒▒▒▒ ▒▒ ▒▒ ▒▒▒▒▒▒ ▒▒ ▒▒ ▒▒▒▒▒▒▒ ",
|
||||
"▓▓ ▓▓ ▓▓ ▓▓ ▓▓ ▓▓ ▓▓ ▓▓ ▓▓ ",
|
||||
"██ ██████ ██ ██ ██ ██ ██ ██ ",
|
||||
"",
|
||||
}
|
||||
|
||||
var bannerColors = []func(a ...any) string{
|
||||
Red,
|
||||
Blue,
|
||||
Red,
|
||||
Blue,
|
||||
Red,
|
||||
}
|
||||
|
||||
var animeArt = []string{
|
||||
"⣿⣿⣿⣿⣿⣷⣿⣿⣿⡅⡹⢿⠆⠙⠋⠉⠻⠿⣿⣿⣿⣿⣿⣿⣮⠻⣦⡙⢷⡑⠘⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣷⣌⠡⠌⠂⣙⠻⣛⠻⠷⠐⠈⠛⢱⣮⣷⣽⣿",
|
||||
"⣿⣿⣿⣿⡇⢿⢹⣿⣶⠐⠁⠀⣀⣠⣤⠄⠀⠀⠈⠙⠻⣿⣿⣿⣦⣵⣌⠻⣷⢝⠦⠚⢿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⢟⣻⣿⣊⡃⠀⣙⠿⣿⣿⣿⣎⢮⡀⢮⣽⣿⣿",
|
||||
"⢿⣿⣿⣿⣧⡸⡎⡛⡩⠖⠀⣴⣿⣿⣿⠀⠀⠀⠀⠸⠇⠀⠙⢿⣿⣿⣿⣷⣌⢷⣑⢷⣄⠻⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⡿⣫⠶⠛⠉⠀⠁⠀⠈⠈⠀⠠⠜⠻⣿⣆⢿⣼⣿⣿⣿",
|
||||
"⢐⣿⣿⣿⣿⣧⢧⣧⢻⣦⢀⣹⣿⣿⣿⣇⠀⠄⠀⠀⠀⡀⠀⠈⢻⣿⣿⣿⣿⣷⣝⢦⡹⠷⡙⢿⣿⣿⣿⣿⣿⣿⣿⣿⠈⠁⠀⠀⠀⠁⠀⠀⠀⠱⣶⣄⡀⠀⠈⠛⠜⣿⣿⣿⣿",
|
||||
"⠀⠊⢫⣿⣏⣿⡌⣼⣄⢫⡌⣿⣿⣿⣿⣿⣦⡈⠲⣄⣤⣤⡡⢀⣠⣿⣿⣿⣿⣿⣿⣷⣼⣍⢬⣦⡙⣿⣿⣿⣿⣿⣯⢁⡄⠀⡀⡀⠀⠄⢈⣠⢪⠀⣿⣿⣿⣦⠀⢉⢂⠹⡿⣿⣿",
|
||||
"⠀⠀⠄⢹⢃⢻⣟⠙⣿⣦⠱⢻⣿⣿⣿⣿⣿⣿⣷⣬⣍⣭⣥⣾⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣶⡙⢿⣼⡿⣿⣿⣿⣿⣿⣷⣄⠘⣱⢦⣤⡴⡿⢈⣼⣿⣿⣿⣇⣴⣶⣮⣅⢻⣿⡏",
|
||||
"⠀⠀⠈⠹⣇⢡⢿⡆⠻⣿⣷⠀⢻⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣷⣍⡻⣿⣟⣻⣿⣿⣿⣿⣷⣦⣥⣬⣤⣴⣾⣿⣿⣿⣿⣷⣿⣿⣿⣿⣷⡜⠃",
|
||||
"⠀⠀⠀⢀⣘⠈⢂⠃⣧⡹⣿⣷⡄⠙⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣮⣅⡙⢿⣟⠿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⠋⡕⠂",
|
||||
"⠀⠀⠀⠀⠀⠀⠛⢷⣜⢷⡌⠻⣿⣿⣦⣝⣻⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣯⣹⣷⣦⣹⢿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⠿⠉⠃⠀",
|
||||
}
|
||||
|
||||
var artColors = []func(a ...any) string{
|
||||
Blue,
|
||||
Blue,
|
||||
Blue,
|
||||
Red,
|
||||
Red,
|
||||
Red,
|
||||
Blue,
|
||||
Blue,
|
||||
Blue,
|
||||
}
|
||||
|
||||
func PrintBanner() {
|
||||
fmt.Println()
|
||||
fmt.Println()
|
||||
for i, line := range portiaBanner {
|
||||
c := bannerColors[i%len(bannerColors)]
|
||||
fmt.Printf(" %s\n", c(line))
|
||||
}
|
||||
fmt.Printf(" %s\n", HiWhite(HRule(52)))
|
||||
fmt.Printf(
|
||||
" %s\n\n",
|
||||
WhiteItalic(
|
||||
"Secrets scanner for git repos and config files",
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
func PrintBannerWithArt() {
|
||||
fmt.Println()
|
||||
for i, line := range portiaBanner {
|
||||
c := bannerColors[i%len(bannerColors)]
|
||||
fmt.Printf(" %s\n", c(line))
|
||||
}
|
||||
fmt.Printf(" %s\n", White(HRule(64)))
|
||||
fmt.Printf(
|
||||
" %s\n",
|
||||
WhiteItalic(
|
||||
"Secrets scanner for git repos and config files",
|
||||
),
|
||||
)
|
||||
fmt.Println()
|
||||
for i, line := range animeArt {
|
||||
c := artColors[i%len(artColors)]
|
||||
fmt.Printf(" %s\n", c(line))
|
||||
}
|
||||
fmt.Println()
|
||||
}
|
||||
|
|
@ -0,0 +1,122 @@
|
|||
// ©AngelaMos | 2026
|
||||
// color.go
|
||||
|
||||
package ui
|
||||
|
||||
import "github.com/fatih/color"
|
||||
|
||||
var (
|
||||
Black = color.New(color.FgBlack).SprintFunc()
|
||||
Red = color.New(color.FgRed).SprintFunc()
|
||||
Green = color.New(color.FgGreen).SprintFunc()
|
||||
Yellow = color.New(color.FgYellow).SprintFunc()
|
||||
Blue = color.New(color.FgBlue).SprintFunc()
|
||||
Magenta = color.New(color.FgMagenta).SprintFunc()
|
||||
Cyan = color.New(color.FgCyan).SprintFunc()
|
||||
White = color.New(color.FgWhite).SprintFunc()
|
||||
|
||||
HiBlack = color.New(color.FgHiBlack).SprintFunc()
|
||||
HiRed = color.New(color.FgHiRed).SprintFunc()
|
||||
HiGreen = color.New(color.FgHiGreen).SprintFunc()
|
||||
HiYellow = color.New(color.FgHiYellow).SprintFunc()
|
||||
HiBlue = color.New(color.FgHiBlue).SprintFunc()
|
||||
HiMagenta = color.New(color.FgHiMagenta).SprintFunc()
|
||||
HiCyan = color.New(color.FgHiCyan).SprintFunc()
|
||||
HiWhite = color.New(color.FgHiWhite).SprintFunc()
|
||||
|
||||
RedBold = color.New(
|
||||
color.FgRed, color.Bold,
|
||||
).SprintFunc()
|
||||
GreenBold = color.New(
|
||||
color.FgGreen, color.Bold,
|
||||
).SprintFunc()
|
||||
YellowBold = color.New(
|
||||
color.FgYellow, color.Bold,
|
||||
).SprintFunc()
|
||||
BlueBold = color.New(
|
||||
color.FgBlue, color.Bold,
|
||||
).SprintFunc()
|
||||
MagentaBold = color.New(
|
||||
color.FgMagenta, color.Bold,
|
||||
).SprintFunc()
|
||||
CyanBold = color.New(
|
||||
color.FgCyan, color.Bold,
|
||||
).SprintFunc()
|
||||
WhiteBold = color.New(
|
||||
color.FgWhite, color.Bold,
|
||||
).SprintFunc()
|
||||
HiRedBold = color.New(
|
||||
color.FgHiRed, color.Bold,
|
||||
).SprintFunc()
|
||||
HiMagentaBold = color.New(
|
||||
color.FgHiMagenta, color.Bold,
|
||||
).SprintFunc()
|
||||
HiBlueBold = color.New(
|
||||
color.FgHiBlue, color.Bold,
|
||||
).SprintFunc()
|
||||
HiCyanBold = color.New(
|
||||
color.FgHiCyan, color.Bold,
|
||||
).SprintFunc()
|
||||
|
||||
Dim = color.New(
|
||||
color.Faint,
|
||||
).SprintFunc()
|
||||
DimItalic = color.New(
|
||||
color.Faint, color.Italic,
|
||||
).SprintFunc()
|
||||
|
||||
CyanItalic = color.New(
|
||||
color.FgCyan, color.Italic,
|
||||
).SprintFunc()
|
||||
MagentaItalic = color.New(
|
||||
color.FgMagenta, color.Italic,
|
||||
).SprintFunc()
|
||||
RedItalic = color.New(
|
||||
color.FgRed, color.Italic,
|
||||
).SprintFunc()
|
||||
GreenItalic = color.New(
|
||||
color.FgGreen, color.Italic,
|
||||
).SprintFunc()
|
||||
WhiteItalic = color.New(
|
||||
color.FgWhite, color.Italic,
|
||||
).SprintFunc()
|
||||
HiBlackItalic = color.New(
|
||||
color.FgHiBlack, color.Italic,
|
||||
).SprintFunc()
|
||||
BlueItalic = color.New(
|
||||
color.FgBlue, color.Italic,
|
||||
).SprintFunc()
|
||||
HiBlueItalic = color.New(
|
||||
color.FgHiBlue, color.Italic,
|
||||
).SprintFunc()
|
||||
YellowItalic = color.New(
|
||||
color.FgYellow, color.Italic,
|
||||
).SprintFunc()
|
||||
|
||||
CyanUnderline = color.New(
|
||||
color.FgCyan, color.Underline,
|
||||
).SprintFunc()
|
||||
MagentaUnderline = color.New(
|
||||
color.FgMagenta, color.Underline,
|
||||
).SprintFunc()
|
||||
HiGreenUnderline = color.New(
|
||||
color.FgHiGreen, color.Underline,
|
||||
).SprintFunc()
|
||||
|
||||
HiCyanItalic = color.New(
|
||||
color.FgHiCyan, color.Italic,
|
||||
).SprintFunc()
|
||||
HiCyanBoldItalic = color.New(
|
||||
color.FgHiCyan, color.Bold, color.Italic,
|
||||
).SprintFunc()
|
||||
HiMagentaBoldItalic = color.New(
|
||||
color.FgHiMagenta, color.Bold, color.Italic,
|
||||
).SprintFunc()
|
||||
HiRedBoldItalic = color.New(
|
||||
color.FgHiRed, color.Bold, color.Italic,
|
||||
).SprintFunc()
|
||||
|
||||
HiBlackCrossed = color.New(
|
||||
color.FgHiBlack, color.CrossedOut,
|
||||
).SprintFunc()
|
||||
)
|
||||
|
|
@ -0,0 +1,86 @@
|
|||
// ©AngelaMos | 2026
|
||||
// spinner.go
|
||||
|
||||
package ui
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
var frames = []string{
|
||||
"\u280b", "\u2819", "\u2839", "\u2838",
|
||||
"\u283c", "\u2834", "\u2826", "\u2827",
|
||||
"\u2807", "\u280f",
|
||||
}
|
||||
|
||||
type Spinner struct {
|
||||
msg string
|
||||
done chan struct{}
|
||||
wg sync.WaitGroup
|
||||
mu sync.Mutex
|
||||
active bool
|
||||
}
|
||||
|
||||
func NewSpinner(msg string) *Spinner {
|
||||
return &Spinner{msg: msg}
|
||||
}
|
||||
|
||||
func (s *Spinner) Start() {
|
||||
s.mu.Lock()
|
||||
if s.active {
|
||||
s.mu.Unlock()
|
||||
return
|
||||
}
|
||||
s.active = true
|
||||
s.done = make(chan struct{})
|
||||
s.wg.Add(1)
|
||||
s.mu.Unlock()
|
||||
|
||||
go s.run()
|
||||
}
|
||||
|
||||
func (s *Spinner) Stop() {
|
||||
s.mu.Lock()
|
||||
if !s.active {
|
||||
s.mu.Unlock()
|
||||
return
|
||||
}
|
||||
s.active = false
|
||||
close(s.done)
|
||||
s.mu.Unlock()
|
||||
s.wg.Wait()
|
||||
}
|
||||
|
||||
func (s *Spinner) run() {
|
||||
defer s.wg.Done()
|
||||
defer fmt.Print("\033[?25h")
|
||||
fmt.Print("\033[?25l")
|
||||
|
||||
ticker := time.NewTicker(80 * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
|
||||
idx := 0
|
||||
for {
|
||||
select {
|
||||
case <-s.done:
|
||||
clearLine()
|
||||
fmt.Print("\033[?25h")
|
||||
return
|
||||
case <-ticker.C:
|
||||
frame := frames[idx%len(frames)]
|
||||
fmt.Printf(
|
||||
"\r %s %s",
|
||||
CyanBold(frame),
|
||||
HiMagenta(s.msg),
|
||||
)
|
||||
idx++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func clearLine() {
|
||||
fmt.Print("\r" + strings.Repeat(" ", 80) + "\r")
|
||||
}
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
// ©AngelaMos | 2026
|
||||
// symbol.go
|
||||
|
||||
package ui
|
||||
|
||||
import "strings"
|
||||
|
||||
const (
|
||||
Arrow = "\u2192"
|
||||
ArrowRight = "\u25b8"
|
||||
ArrowUp = "\u2191"
|
||||
Diamond = "\u25c6"
|
||||
Gem = "\u25c8"
|
||||
Star = "\u2726"
|
||||
TriangleUp = "\u25b2"
|
||||
Check = "\u2713"
|
||||
Cross = "\u2717"
|
||||
Timer = "\u23f1"
|
||||
Warning = "\u26a0"
|
||||
Shield = "\u25c9"
|
||||
DividerChar = "\u2501"
|
||||
)
|
||||
|
||||
func HRule(width int) string {
|
||||
return strings.Repeat(DividerChar, width)
|
||||
}
|
||||
|
|
@ -0,0 +1,223 @@
|
|||
# Portia - Secrets Scanner for Codebases
|
||||
|
||||
## What This Is
|
||||
|
||||
Portia is a Go CLI tool that scans source code directories and git history for leaked secrets: API keys, passwords, tokens, private keys, connection strings. It uses 150 detection rules, Shannon entropy analysis, a 5-layer false positive filter, and optional Have I Been Pwned breach verification. Files are split into 50-line chunks and processed through a bounded concurrent worker pool, with results reported in terminal, JSON, or SARIF format.
|
||||
|
||||
The name comes from Shakespeare's *The Merchant of Venice*, where Portia is the one who exposes what's hidden in the contract. This tool does the same for your codebase.
|
||||
|
||||
## Why This Matters
|
||||
|
||||
Hardcoded secrets in source code are one of the most common and most damaging security mistakes in software development. This isn't theoretical. Here are three incidents that show exactly why this matters:
|
||||
|
||||
**Uber, 2016 - Hardcoded AWS Keys on GitHub**
|
||||
Two Uber engineers committed AWS access keys to a private GitHub repository. Attackers who gained access to that repo used the keys to reach an S3 bucket containing the personal information of 57 million riders and drivers. Uber paid $148 million in settlements. The keys were sitting in the repo for months before the breach. A tool like Portia, scanning on every commit, would have flagged `AKIA...` patterns immediately using the `aws-access-key-id` rule defined at `internal/rules/builtin.go:22-31`. The regex `\b((?:AKIA|ABIA|ACCA|ASIA)[0-9A-Z]{16})\b` catches exactly this format.
|
||||
|
||||
**CircleCI, January 2023 - Secrets in CI Configuration**
|
||||
An attacker compromised a CircleCI engineer's laptop and used the session cookie to access internal systems. They then extracted customer secrets that were stored as environment variables in CI/CD pipelines. The advisory (CCI-2023-001) urged all customers to rotate every secret stored in CircleCI. The root issue: secrets were configured as plaintext values in CI configs rather than fetched from vaults at runtime. If those config files had been scanned before deployment, the problem would have been caught. Portia's generic rules for `password`, `secret`, `token`, and `api_key` assignments target exactly this pattern.
|
||||
|
||||
**Codecov, April 2021 - Supply Chain Attack via Exfiltrated Tokens**
|
||||
Attackers modified Codecov's bash uploader script to exfiltrate environment variables from CI environments. Every company using Codecov's bash uploader leaked whatever tokens and keys were present in their CI environment, including credentials for GitHub, AWS, and internal services. The modified script ran for two months before detection. Among the affected companies: Twitch, HashiCorp, Confluent. Portia detects tokens for GitHub (`ghp_`, `gho_`, `ghs_`), HashiCorp Vault (`hvs.`, `hvb.`, `hvr.`), and many other providers, all defined in `internal/rules/builtin.go`.
|
||||
|
||||
The common thread: secrets sitting in code or configs where automated tools could have caught them. That's what this project builds.
|
||||
|
||||
## What You'll Learn
|
||||
|
||||
Building this project teaches you both security concepts and real Go engineering patterns.
|
||||
|
||||
**Security Concepts:**
|
||||
- **Secret sprawl and why it happens** - Why developers keep hardcoding credentials despite knowing better, and the systemic solutions (environment variables, vault integrations, pre-commit hooks)
|
||||
- **Shannon entropy for anomaly detection** - Using information theory to distinguish random API keys from normal English text. The math is at `internal/rules/entropy.go`.
|
||||
- **k-Anonymity** - Troy Hunt's protocol for checking if a secret appears in breach databases without revealing the secret itself. Implementation at `internal/hibp/client.go`.
|
||||
- **False positive reduction** - Why naive regex scanning produces thousands of garbage results, and the 5-layer filtering approach that makes output actually useful. See `internal/engine/filter.go`.
|
||||
- **SARIF for CI integration** - The industry-standard format that GitHub, Azure DevOps, and other CI platforms consume to show security findings inline on pull requests.
|
||||
|
||||
**Technical Skills (Go-specific):**
|
||||
- **errgroup concurrent pipelines** - Bounded worker pools using `golang.org/x/sync/errgroup` with channel-based communication. The full pipeline is at `internal/engine/pipeline.go`.
|
||||
- **Cobra CLI framework** - Building a multi-command CLI with persistent flags, config file support, and subcommands. Root command at `internal/cli/root.go`.
|
||||
- **go-git library** - Programmatic git history traversal, commit walking, blob reading, and staged file scanning without shelling out to `git`. See `internal/source/git.go`.
|
||||
- **Circuit breaker pattern** - Using `gobreaker` to prevent cascading failures when external APIs (HIBP) are down. Configured at `internal/hibp/client.go`.
|
||||
- **LRU caching** - Bounded in-memory caching with `hashicorp/golang-lru` to avoid redundant API calls. See `internal/hibp/client.go`.
|
||||
- **TOML configuration** - Layered config resolution with CLI flags > config file > defaults. Supports both `.portia.toml` and `pyproject.toml` (`[tool.portia]` table). See `internal/config/config.go` and `internal/cli/root.go`.
|
||||
|
||||
**Tools and Techniques:**
|
||||
- **Regex crafting for security** - Writing patterns that match real credential formats (AWS key prefixes, GitHub token formats, Stripe key patterns) while avoiding false positives
|
||||
- **just task runner** - Using `Justfile` as an alternative to Makefiles for build, test, lint workflows
|
||||
- **golangci-lint** - Static analysis configuration at `.golangci.yml` for enforcing code quality
|
||||
|
||||
## Prerequisites
|
||||
|
||||
**Required knowledge:**
|
||||
- **Go basics** - You need to understand goroutines, channels, interfaces, and structs. The pipeline uses all of these heavily. If `go func()` or `chan types.Chunk` looks unfamiliar, work through the Go Tour first.
|
||||
- **Regular expressions** - The detection engine is built on regex. You should know capture groups, quantifiers, character classes, and non-capturing groups. The rules at `internal/rules/builtin.go` use all of these.
|
||||
- **Git fundamentals** - You should understand commits, branches, staging, and blobs. The git scanner walks these structures programmatically.
|
||||
|
||||
**Tools you'll need:**
|
||||
- **Go 1.22+** - Uses `range` over integers (e.g., `for range p.workers` at `internal/engine/pipeline.go:53`), which requires Go 1.22
|
||||
- **just** - Task runner. Install with `cargo install just` or your package manager
|
||||
- **git** - For the git history scanning features
|
||||
|
||||
**Helpful but not required:**
|
||||
- **Information theory** - Understanding entropy helps, but the concepts doc explains it from scratch
|
||||
- **SARIF specification** - The OASIS standard for static analysis results. You can read the output without knowing the spec.
|
||||
|
||||
## Installation
|
||||
|
||||
Three ways to install Portia:
|
||||
|
||||
**Option 1: Install script** (no Go required)
|
||||
```bash
|
||||
curl -fsSL https://raw.githubusercontent.com/CarterPerez-dev/portia/main/install.sh | bash
|
||||
```
|
||||
|
||||
This downloads a pre-built binary for your platform (Linux/macOS, amd64/arm64). If no binary is available, it falls back to building from source with Go.
|
||||
|
||||
**Option 2: Go install**
|
||||
```bash
|
||||
go install github.com/CarterPerez-dev/portia/cmd/portia@latest
|
||||
```
|
||||
|
||||
Requires Go 1.24+. The binary is placed in your `$GOPATH/bin` (or `$GOBIN`).
|
||||
|
||||
**Option 3: Build from source**
|
||||
```bash
|
||||
cd PROJECTS/intermediate/secrets-scanner
|
||||
go build -o portia ./cmd/portia
|
||||
```
|
||||
|
||||
This is the path you'll use when working through this project and making changes to the code.
|
||||
|
||||
## Quick Start
|
||||
|
||||
Get the project running:
|
||||
```bash
|
||||
cd PROJECTS/intermediate/secrets-scanner
|
||||
|
||||
./portia scan ./testdata/fixtures
|
||||
|
||||
./portia scan --format json ./testdata/fixtures
|
||||
|
||||
./portia scan --hibp ./testdata/fixtures
|
||||
|
||||
./portia git --depth 10 .
|
||||
|
||||
./portia git --staged .
|
||||
```
|
||||
|
||||
Expected output on the test fixtures: colored terminal output showing detected secrets with severity levels, rule IDs, file paths, line numbers, entropy scores, and masked secret values.
|
||||
|
||||
To initialize a config file:
|
||||
```bash
|
||||
./portia init
|
||||
```
|
||||
|
||||
This creates `.portia.toml` with default settings. Edit it to disable rules, set excludes, or enable HIBP checking by default.
|
||||
|
||||
For Python projects, you can use `pyproject.toml` instead:
|
||||
```bash
|
||||
./portia pyproject
|
||||
```
|
||||
|
||||
This creates a `pyproject.toml` with a `[tool.portia]` section. Portia automatically reads from `pyproject.toml` when no `.portia.toml` is found.
|
||||
|
||||
## Project Structure
|
||||
|
||||
```
|
||||
secrets-scanner/
|
||||
├── cmd/portia/main.go # Entry point, calls cli.Execute()
|
||||
├── internal/
|
||||
│ ├── cli/ # Cobra commands
|
||||
│ │ ├── root.go # Root command, flag definitions, config init
|
||||
│ │ ├── scan.go # `portia scan` - directory scanning
|
||||
│ │ ├── git.go # `portia git` - git history scanning
|
||||
│ │ ├── init.go # `portia init` + `portia pyproject` - create config files
|
||||
│ │ └── config.go # `portia config` - show current config
|
||||
│ ├── config/
|
||||
│ │ └── config.go # TOML config loader, defaults, search paths
|
||||
│ ├── engine/
|
||||
│ │ ├── detector.go # Applies rules to chunks, entropy validation
|
||||
│ │ ├── filter.go # Stopwords, placeholders, templates, allowlists
|
||||
│ │ └── pipeline.go # errgroup worker pool, deduplication
|
||||
│ ├── hibp/
|
||||
│ │ └── client.go # k-anonymity API client, LRU cache, circuit breaker
|
||||
│ ├── reporter/
|
||||
│ │ ├── reporter.go # Reporter interface and factory
|
||||
│ │ ├── terminal.go # Colored terminal output with masking
|
||||
│ │ ├── json.go # JSON output format
|
||||
│ │ └── sarif.go # SARIF v2.1.0 output for CI integration
|
||||
│ ├── rules/
|
||||
│ │ ├── builtin.go # 150 detection rules covering AWS, GitHub, etc.
|
||||
│ │ ├── entropy.go # Shannon entropy calculator and charset detection
|
||||
│ │ └── registry.go # Rule storage, keyword matching, global allowlists
|
||||
│ ├── source/
|
||||
│ │ ├── source.go # Source interface definition
|
||||
│ │ ├── directory.go # Filesystem walker with 50-line chunking
|
||||
│ │ └── git.go # Git history scanner using go-git
|
||||
│ └── ui/ # Colors, spinner, banner, symbols
|
||||
├── pkg/types/types.go # Core types: Finding, Chunk, Rule, Severity
|
||||
├── testdata/fixtures/ # Test secrets for integration testing
|
||||
├── Justfile # Task runner
|
||||
└── .golangci.yml # Linter configuration
|
||||
```
|
||||
|
||||
## Development Commands
|
||||
|
||||
This project uses [`just`](https://github.com/casey/just) as a command runner. Run `just` with no arguments to see all available commands.
|
||||
|
||||
| Command | Description |
|
||||
|---------|-------------|
|
||||
| `just lint` | Run golangci-lint |
|
||||
| `just lint-fix` | Run golangci-lint with auto-fix |
|
||||
| `just format` | Format code via golangci-lint |
|
||||
| `just vet` | Run `go vet` |
|
||||
| `just tidy` | Run `go mod tidy` |
|
||||
| `just test` | Run all tests with race detector |
|
||||
| `just test-v` | Run tests with verbose output |
|
||||
| `just cover` | Run tests with coverage summary |
|
||||
| `just cover-html` | Generate HTML coverage report |
|
||||
| `just ci` | Run lint + test (full CI check) |
|
||||
| `just check` | Run lint + vet |
|
||||
| `just run <args>` | Run portia with arguments (e.g., `just run scan .`) |
|
||||
| `just dev-scan` | Scan testdata directory |
|
||||
| `just dev-git` | Scan current repo git history |
|
||||
| `just dev-json` | Scan testdata with JSON output |
|
||||
| `just dev-sarif` | Scan testdata with SARIF output |
|
||||
| `just dev-rules` | List all detection rules |
|
||||
| `just build` | Production build to `bin/portia` |
|
||||
| `just install` | `go install` to `$GOPATH/bin` |
|
||||
| `just info` | Show project/Go/OS info |
|
||||
| `just clean` | Remove build artifacts |
|
||||
|
||||
## Next Steps
|
||||
|
||||
Work through the documents in order:
|
||||
|
||||
1. **01-CONCEPTS.md** - Security concepts: secret sprawl, Shannon entropy, regex-based detection, false positive filtering, k-anonymity, SARIF
|
||||
2. **02-ARCHITECTURE.md** - System design: pipeline architecture, concurrency model, data flow, component interactions
|
||||
3. **03-IMPLEMENTATION.md** - Code walkthrough: every key file explained with line references
|
||||
4. **04-CHALLENGES.md** - Extensions: pre-commit hooks, custom rules, incremental scanning, GitHub Actions
|
||||
|
||||
## Common Issues
|
||||
|
||||
**"No secrets detected" on real code:**
|
||||
Your code might be clean. Try scanning `testdata/fixtures/` first to confirm the tool works. If scanning real code, check that you're not excluding too many paths. Use `--verbose` to see which files are being scanned.
|
||||
|
||||
**Git scan is slow:**
|
||||
Use `--depth` to limit how many commits to scan. `--depth 100` scans the last 100 commits instead of the entire history. For a quick check, use `--staged` to scan only staged changes.
|
||||
|
||||
**Too many false positives:**
|
||||
Edit `.portia.toml` to add paths or values to the allowlist. You can also disable specific rules with `rules.disable = ["generic-password"]`. The filter system (stopwords, placeholders, templates) handles most cases, but project-specific patterns may need custom allowlisting.
|
||||
|
||||
**HIBP checking fails:**
|
||||
The HIBP API has rate limits. The circuit breaker at `internal/hibp/client.go` will open after 5 consecutive failures, preventing further requests for 60 seconds. This is by design. If the API is consistently unavailable, the scan still completes without breach data.
|
||||
|
||||
**Build errors with Go version:**
|
||||
This project uses `for range p.workers` syntax which requires Go 1.22+. Check your version with `go version`.
|
||||
|
||||
## Related Projects
|
||||
|
||||
Other projects in this repository that complement secrets scanning:
|
||||
|
||||
- **api-security-scanner** (`PROJECTS/intermediate/api-security-scanner/`) - Scans APIs for security misconfigurations. Where Portia finds secrets in code, this project tests whether APIs are properly secured at runtime.
|
||||
- **docker-security-audit** (`PROJECTS/intermediate/docker-security-audit/`) - Audits Docker images and configs for security issues. Complements Portia by checking container environments where secrets are often exposed via environment variables.
|
||||
- **api-rate-limiter** (`PROJECTS/advanced/api-rate-limiter/`) - Implements rate limiting algorithms. Relevant because Portia's HIBP client uses the same circuit breaker pattern to handle external API rate limits gracefully.
|
||||
|
|
@ -0,0 +1,237 @@
|
|||
# Core Security Concepts
|
||||
|
||||
This document explains the security concepts you'll encounter while building Portia. These aren't just definitions. We'll dig into why they matter and how they actually work in production systems.
|
||||
|
||||
## Secret Sprawl
|
||||
|
||||
### What It Is
|
||||
|
||||
Secret sprawl is the uncontrolled spread of credentials, API keys, tokens, and other sensitive values across source code, configuration files, CI/CD pipelines, chat logs, and documentation. Once a secret enters a git repository, it exists in every clone forever, even if the file is later deleted.
|
||||
|
||||
### Why It Matters
|
||||
|
||||
GitHub reported in 2023 that they detected over 12.8 million new secrets leaked in public repositories that year. That number only covers the secrets they actively scan for. The real number is significantly higher.
|
||||
|
||||
**Uber, November 2016**
|
||||
Two engineers committed AWS access keys to a private GitHub repository. Six months later, attackers who had access to that repo used the keys to access an S3 bucket containing 57 million rider and driver records (names, email addresses, phone numbers, driver license numbers). Uber paid the attackers $100,000 to delete the data and keep quiet, then paid $148 million in a settlement with all 50 US state attorneys general. The keys matched the exact `AKIA` prefix pattern that Portia's `aws-access-key-id` rule detects (`internal/rules/builtin.go`).
|
||||
|
||||
**Samsung, March 2022**
|
||||
The Lapsus$ group exfiltrated 190GB of Samsung source code from internal Git servers. Inside the code: hardcoded credentials for Samsung's SmartThings platform, private signing keys, and source code for bootloaders and TrustZone. The attack surface existed because secrets were embedded directly in application code rather than injected at runtime.
|
||||
|
||||
**Twitter, January 2023**
|
||||
A researcher discovered that Twitter's mobile apps contained embedded API keys that could be extracted through reverse engineering. Twitter acknowledged the issue and rotated the keys, but the incident highlighted a fundamental problem: mobile apps ship compiled code to millions of devices, and any secrets in that code are extractable.
|
||||
|
||||
### How It Works
|
||||
|
||||
Secrets end up in code through three main paths:
|
||||
|
||||
**Development shortcuts**: A developer needs an API key to test a feature. They paste it into a config file, commit it, and plan to move it to environment variables later. They never do. The key sits in git history forever. Even deleting the file and committing the deletion doesn't help because `git log --all --full-history` still shows it.
|
||||
|
||||
**Copy-paste from documentation**: Cloud providers often show example commands with placeholder keys like `AKIAIOSFODNN7EXAMPLE`. Developers replace the placeholder with their real key, forget to revert, and commit the file.
|
||||
|
||||
**CI/CD configuration**: Secrets are set as environment variables in CI pipelines (GitHub Actions, CircleCI, Jenkins). When a developer copies a workflow file or exports a pipeline config, those values end up in version-controlled files. The CircleCI breach in January 2023 happened partly because customer secrets were stored as plaintext environment variables accessible to CI jobs.
|
||||
|
||||
### Defense Strategies
|
||||
|
||||
Portia implements pre-commit scanning, which catches secrets before they enter git history. But scanning is just one layer:
|
||||
|
||||
1. **Pre-commit hooks** - Run Portia on staged files before every commit
|
||||
2. **CI pipeline scanning** - Run Portia on every pull request as a required check
|
||||
3. **Vault integration** - Use HashiCorp Vault, AWS Secrets Manager, or similar tools to inject secrets at runtime
|
||||
4. **Environment variables** - Keep secrets out of code entirely. Reference `${API_KEY}` instead of pasting the key.
|
||||
5. **Rotation** - Assume secrets will leak. Rotate them regularly so leaked keys expire quickly.
|
||||
|
||||
## Shannon Entropy
|
||||
|
||||
### What It Is
|
||||
|
||||
Shannon entropy (named after Claude Shannon, the father of information theory) measures the randomness of a string. It answers the question: "How surprised would I be by each character?" English text has low entropy (you can predict the next letter). An API key like `xK9mP2vL5nQ8jR3t` has high entropy (each character is unpredictable).
|
||||
|
||||
The formula: `H = -Σ p(x) * log₂(p(x))`
|
||||
|
||||
Where `p(x)` is the frequency of each character divided by the string length.
|
||||
|
||||
### Why It Matters
|
||||
|
||||
Regex alone isn't enough to detect all secrets. Some rules match structural patterns (`AKIA` prefix for AWS keys, `ghp_` prefix for GitHub tokens). But many secrets have no structural prefix. A password like `xK9mP2vL5nQ8jR3t` looks like random characters, and that randomness itself is the signal.
|
||||
|
||||
Human-readable text (variable names, comments, URLs) has entropy around 2.5-3.5 bits per character. Randomly generated secrets typically have entropy above 4.0. By computing entropy and comparing to a threshold, Portia can flag high-randomness strings that don't match any specific regex pattern.
|
||||
|
||||
### How It Works
|
||||
|
||||
The implementation at `internal/rules/entropy.go` works in three steps:
|
||||
|
||||
**Step 1: Charset detection** - Determine whether the string looks like hex (`0-9a-f`), base64 (`A-Za-z0-9+/=`), or general alphanumeric. This matters because the entropy threshold differs: a hex string needs lower entropy to be suspicious (since hex only has 16 possible characters), while alphanumeric strings need higher entropy.
|
||||
|
||||
**Step 2: Character frequency counting** - Count how many times each character appears. For `aabbcc`, the frequencies are: `a=2, b=2, c=2`. Each character has probability 2/6 = 0.333.
|
||||
|
||||
**Step 3: Entropy calculation** - For each unique character, compute `-p * log₂(p)` and sum them. For `aabbcc`: `-0.333 * log₂(0.333) * 3 unique chars = 1.585 bits`. For a random 20-character string with all unique characters: `-0.05 * log₂(0.05) * 20 = 4.322 bits`.
|
||||
|
||||
**Thresholds used in Portia:**
|
||||
- Hex strings (charset `0-9a-f`): 3.0 bits minimum
|
||||
- Base64 strings (charset `A-Za-z0-9+/=`): 4.0 bits minimum
|
||||
- Alphanumeric strings: 3.5 bits minimum
|
||||
|
||||
These thresholds are configured per-rule in `internal/rules/builtin.go`. Rules like `generic-password` and `generic-secret` use entropy thresholds to filter out low-randomness false positives like `password = "admin"`.
|
||||
|
||||
### Common Pitfalls
|
||||
|
||||
**Low entropy doesn't mean safe.** The string `aaaaaaaaAAAAAAAA` has low entropy but could still be a valid API key for a poorly designed system. Entropy is one signal, not the only signal.
|
||||
|
||||
**Charset matters.** A hex string `deadbeef` has low absolute entropy but might be suspicious in a hex context. Portia's `DetectCharset` function (`internal/rules/entropy.go`) adjusts thresholds based on the character set.
|
||||
|
||||
## Regular Expression-Based Detection
|
||||
|
||||
### What It Is
|
||||
|
||||
Most cloud providers and services use structured formats for their credentials. AWS access keys always start with `AKIA`, `ABIA`, `ACCA`, or `ASIA` followed by 16 uppercase alphanumeric characters. GitHub PATs start with `ghp_` followed by 36 alphanumeric characters. Stripe live keys start with `sk_live_`. These patterns are specific enough to match with regular expressions while having very low false positive rates.
|
||||
|
||||
### Why It Matters
|
||||
|
||||
Prefix-based detection works because cloud providers intentionally design their key formats to be identifiable. AWS uses the `AKIA` prefix specifically so that tools can detect leaked keys. GitHub changed their token format from random hex strings to prefixed tokens (`ghp_`, `gho_`, `ghs_`, `ghr_`) in 2021 to enable exactly this kind of scanning.
|
||||
|
||||
### How It Works
|
||||
|
||||
Each rule in `internal/rules/builtin.go` has four components that work together:
|
||||
|
||||
**Keywords** - Fast string matching to eliminate ~95% of chunks before regex. If a chunk doesn't contain the keyword `AKIA`, there's no point running the AWS key regex against it. This is a performance optimization. See `internal/rules/registry.go` MatchKeywords function.
|
||||
|
||||
**Pattern** - The actual regex. For AWS keys: `\b((?:AKIA|ABIA|ACCA|ASIA)[0-9A-Z]{16})\b`. The `\b` word boundaries prevent matching substrings of longer tokens. The capture group `(...)` extracts just the secret portion.
|
||||
|
||||
**SecretGroup** - Which regex capture group contains the secret value. Group 0 is the entire match; group 1 is the first parenthesized group. Most rules use group 1 to extract the key without surrounding context.
|
||||
|
||||
**Allowlist** - Per-rule patterns that suppress known false positives. AWS keys have an allowlist for `AKIAIOSFODNN7EXAMPLE` (the AWS documentation example key).
|
||||
|
||||
### 150 Rules Organized by Provider
|
||||
|
||||
Portia's builtin rules cover:
|
||||
- **Cloud providers**: AWS (3), GCP (3), Azure (3), Alibaba Cloud (1), IBM Cloud (1), Cloudflare (2)
|
||||
- **Source control**: GitHub (6), GitLab (3), Bitbucket (1)
|
||||
- **Payment**: Stripe (4), Square (2), Razorpay (1), Braintree (1), Coinbase (1)
|
||||
- **Communication**: Slack (4), Twilio (3), SendGrid (1), Discord (3), Telegram (1), Microsoft Teams (1), Intercom (1)
|
||||
- **Email**: Mailchimp (1), Mailgun (1), Resend (1), Brevo (1), Postmark (1)
|
||||
- **Infrastructure**: Heroku (1), DigitalOcean (3), Supabase (2), Confluent (1), Fly.io (1), Render (1), Vercel (1), Netlify (1), PlanetScale (3), Neon (1), Upstash (1), Turso (1)
|
||||
- **Hosting/VPS**: Hetzner (1), Linode (1), Vultr (1)
|
||||
- **AI/ML**: OpenAI (2), Anthropic (1), HuggingFace (1), Replicate (1), Groq (1), Perplexity (1)
|
||||
- **CI/CD**: CircleCI (1), Buildkite (1), GitHub Actions (1)
|
||||
- **Secrets management**: Vault (1), Doppler (2), 1Password (1)
|
||||
- **Cryptographic**: SSH/PGP/PKCS8 keys (6), JWT (1), Age (1)
|
||||
- **Package registries**: NPM (1), PyPI (1), RubyGems (1), Docker Hub (1)
|
||||
- **Generic**: password (1), secret (1), api-key (1), token (1)
|
||||
- **Connection strings**: PostgreSQL (1), MySQL (1), MongoDB (1), Redis (1), Firebase URL (1), Cloudinary (1)
|
||||
- **Monitoring/Observability**: Datadog (1), New Relic (1), Grafana (2), Sentry (1), PostHog (1), Axiom (1), Dynatrace (1), Honeycomb (1), Elastic (1), Segment (1), Rollbar (1), Mixpanel (1), Amplitude (1)
|
||||
- **Developer tools**: Figma (1), Linear (1), Postman (1), Algolia (1), Contentful (1), Snyk (1), SonarQube (1), Freshdesk (1), Zendesk (1)
|
||||
- **Other**: Terraform (1), Shopify (4), Okta (1), LaunchDarkly (2), Infracost (1), Prefect (1), Pulumi (1), Databricks (1), HubSpot (1), PagerDuty (1), Atlassian (1), Facebook (1), Twitter (1), Firebase FCM (1), Mapbox (2), Doppler CLI (1)
|
||||
|
||||
## False Positive Reduction
|
||||
|
||||
### What It Is
|
||||
|
||||
A naive regex scanner produces hundreds or thousands of false positives: placeholder values, template variables, test fixtures, example keys from documentation. If the output is noisy, developers ignore it. False positive reduction is the set of techniques that make scanner output trustworthy enough to act on.
|
||||
|
||||
### Why It Matters
|
||||
|
||||
GitGuardian's 2023 State of Secrets Sprawl report found that teams using secrets scanners with high false positive rates had a 40% lower remediation rate than teams with low false positive scanners. Noise kills action. If every other finding is a false positive, developers stop reading findings.
|
||||
|
||||
### How It Works
|
||||
|
||||
Portia uses a 5-layer filter chain at `internal/engine/filter.go` and `internal/engine/detector.go`:
|
||||
|
||||
**Layer 1: Keyword pre-filtering** (`internal/rules/registry.go` MatchKeywords)
|
||||
Before running any regex, check if the chunk even contains keywords relevant to any rule. A file full of HTML with no instance of `password`, `secret`, `key`, `token`, or any provider-specific prefix won't match any rules. This eliminates ~95% of chunks immediately and is a pure performance optimization.
|
||||
|
||||
**Layer 2: Structural validation** (`internal/engine/detector.go` extractSecret)
|
||||
After regex matches, extract the secret using the capture group. If the capture group is empty or the match is just whitespace, discard it. This handles edge cases where a regex matches surrounding syntax but captures nothing meaningful.
|
||||
|
||||
**Layer 3: Placeholder detection** (`internal/engine/filter.go` IsPlaceholder)
|
||||
Check the secret value against known placeholder patterns: `example`, `test`, `dummy`, `fake`, `placeholder`, `YOUR_API_KEY`, `xxxx...`, `****...`, `${VARIABLE}`, `{{TEMPLATE}}`, `TODO`, `CHANGEME`, `REPLACE_ME`. These are defined in `internal/rules/registry.go` GlobalValueAllowlist.
|
||||
|
||||
**Layer 4: Template detection** (`internal/engine/filter.go` IsTemplated)
|
||||
Check if the secret is a template variable: `${ENV_VAR}`, `{{handlebars}}`, `os.getenv(...)`, `process.env.X`, `System.getenv(...)`, `ENV[...]`. These aren't real secrets; they're references to secrets stored elsewhere.
|
||||
|
||||
**Layer 5: Stopword filtering** (`internal/engine/filter.go` IsStopword)
|
||||
Check if the secret, when split on delimiters (`_`, `-`, `.`, `/`), contains common programming words like `function`, `controller`, `database`, `config`, etc. The stopword list at `internal/engine/filter.go` has 700+ words. This catches false positives like `module_controller_factory` that regex might match as a generic token.
|
||||
|
||||
**Layer 6: Path allowlisting** (`internal/rules/registry.go` GlobalPathAllowlist)
|
||||
Skip files that are known to contain non-sensitive data: `go.mod`, `go.sum`, `package-lock.json`, `yarn.lock`, `.min.js` files, `node_modules/`, `vendor/`, binary formats.
|
||||
|
||||
### Common Pitfall
|
||||
|
||||
The biggest mistake with false positive reduction: being too aggressive. If you filter out too many results, you'll miss real secrets. Portia's approach is conservative by default. Each filter layer targets a specific class of false positives with high precision. The stopword list only matches on delimiter-split parts (not substrings) to avoid filtering out real secrets that happen to contain common words as substrings.
|
||||
|
||||
## HIBP k-Anonymity
|
||||
|
||||
### What It Is
|
||||
|
||||
Have I Been Pwned (HIBP) is Troy Hunt's database of billions of passwords and credentials from known data breaches. Portia can check whether a detected secret appears in any known breach. But sending the secret to an external API would defeat the purpose. k-Anonymity solves this: you send only 5 characters of the secret's SHA-1 hash, and the server returns all hashes that share that prefix. You check locally whether your full hash appears in the results.
|
||||
|
||||
### Why It Matters
|
||||
|
||||
If Portia finds `password = "P@ssw0rd123"` in your code, knowing it appears in 47,000 known breaches adds urgency. A unique, randomly-generated password that hasn't been breached is less urgent than a commonly-used password that's in every credential stuffing wordlist.
|
||||
|
||||
The 2019 Collection #1 breach dump contained 773 million unique email addresses and 21 million unique passwords. The 2021 RockYou2021 leak contained 8.4 billion passwords. If your detected secret appears in these datasets, attackers already have it.
|
||||
|
||||
### How It Works
|
||||
|
||||
Implementation at `internal/hibp/client.go`:
|
||||
|
||||
1. **SHA-1 hash**: Compute `SHA-1(secret)`. For "P@ssw0rd123": `a94a8fe5ccb19ba61c4c0873d391e987982fbbd3`
|
||||
2. **Prefix extraction**: Take the first 5 hex characters: `a94a8`
|
||||
3. **API query**: `GET https://api.pwnedpasswords.com/range/a94a8`
|
||||
4. **Response**: The API returns ~500-800 hash suffixes with occurrence counts
|
||||
5. **Local matching**: Check if the remaining 35 characters of your hash appear in the response
|
||||
|
||||
The API never sees the full hash, so it can't determine which password you're checking. This is the k-anonymity guarantee.
|
||||
|
||||
**LRU cache** (`internal/hibp/client.go`): Results are cached in a 10,000-entry LRU cache. If the same prefix has been queried before, the cached result is returned. This avoids redundant API calls when scanning large codebases with similar secrets.
|
||||
|
||||
**Circuit breaker** (`internal/hibp/client.go`): Using Sony's gobreaker library, the client tracks API failures. After 5 consecutive errors, the circuit opens and requests are rejected immediately for 60 seconds. This prevents cascading failures if the HIBP API is down or rate-limited.
|
||||
|
||||
### Common Pitfall
|
||||
|
||||
HIBP is designed for passwords, not API keys. A leaked AWS access key won't appear in password breach databases. HIBP checking is most useful for password-type findings (the `generic-password` and `generic-secret` rules) and less useful for provider-specific tokens. Portia only sends password and generic-secret findings to HIBP, skipping provider-specific tokens entirely.
|
||||
|
||||
## SARIF Output Format
|
||||
|
||||
### What It Is
|
||||
|
||||
SARIF (Static Analysis Results Interchange Format) is an OASIS standard (version 2.1.0) for expressing the output of static analysis tools. It's a JSON-based format that GitHub, Azure DevOps, GitLab, and other CI platforms can consume to display findings inline on pull requests.
|
||||
|
||||
### Why It Matters
|
||||
|
||||
Without SARIF, integrating a scanner into CI requires custom parsing logic for each tool's output format. With SARIF, you produce one JSON file and every major CI platform knows how to display it. GitHub Code Scanning, for example, accepts SARIF uploads and shows findings as annotations on the PR diff.
|
||||
|
||||
### How It Works
|
||||
|
||||
Portia's SARIF reporter at `internal/reporter/sarif.go` produces a v2.1.0 compliant document:
|
||||
|
||||
```json
|
||||
{
|
||||
"$schema": "https://json.schemastore.org/sarif-2.1.0.json",
|
||||
"version": "2.1.0",
|
||||
"runs": [{
|
||||
"tool": {
|
||||
"driver": {
|
||||
"name": "portia",
|
||||
"rules": [...]
|
||||
}
|
||||
},
|
||||
"results": [...]
|
||||
}]
|
||||
}
|
||||
```
|
||||
|
||||
Each finding maps to a SARIF `result` with:
|
||||
- `ruleId` - The Portia rule ID (e.g., `aws-access-key-id`)
|
||||
- `level` - Mapped from Portia severity (CRITICAL/HIGH = `error`, MEDIUM = `warning`, LOW = `note`)
|
||||
- `message` - The finding description
|
||||
- `locations` - File path and line number
|
||||
- `properties` - Portia-specific metadata (entropy, HIBP status, masked secret)
|
||||
|
||||
To upload to GitHub Code Scanning:
|
||||
```bash
|
||||
portia scan --format sarif . > results.sarif
|
||||
gh api repos/{owner}/{repo}/code-scanning/sarifs \
|
||||
-f sarif=@results.sarif \
|
||||
-f commit_sha=$(git rev-parse HEAD)
|
||||
```
|
||||
|
|
@ -0,0 +1,328 @@
|
|||
# System Architecture
|
||||
|
||||
This document breaks down how Portia is designed and why certain architectural decisions were made. We'll trace requests through the system and explain the tradeoffs.
|
||||
|
||||
## High Level Architecture
|
||||
|
||||
```
|
||||
┌──────────────────────────────────────────────────────┐
|
||||
│ CLI │
|
||||
│ root.go, scan.go, git.go │
|
||||
└───────────────────────┬──────────────────────────────┘
|
||||
│
|
||||
┌─────────┴─────────┐
|
||||
▼ ▼
|
||||
┌──────────────┐ ┌──────────────┐
|
||||
│ Directory │ │ Git │
|
||||
│ Source │ │ Source │
|
||||
│ directory.go │ │ git.go │
|
||||
└──────┬───────┘ └──────┬───────┘
|
||||
│ │
|
||||
└─────────┬─────────┘
|
||||
│
|
||||
▼ chan types.Chunk
|
||||
┌─────────────────┐
|
||||
│ Pipeline │
|
||||
│ pipeline.go │
|
||||
├─────────────────┤
|
||||
│ │
|
||||
│ ┌───────────┐ │
|
||||
│ │ Worker 1 │ │
|
||||
│ │ detector │ │
|
||||
│ └───────────┘ │
|
||||
│ ┌───────────┐ │
|
||||
│ │ Worker 2 │ │
|
||||
│ │ detector │ │
|
||||
│ └───────────┘ │
|
||||
│ ┌───────────┐ │
|
||||
│ │ Worker N │ │
|
||||
│ │ detector │ │
|
||||
│ └───────────┘ │
|
||||
│ │
|
||||
└────────┬────────┘
|
||||
│
|
||||
▼ chan types.Finding
|
||||
┌─────────────────┐
|
||||
│ Collector │
|
||||
│ dedup + merge │
|
||||
└────────┬────────┘
|
||||
│
|
||||
┌────────┴────────┐
|
||||
│ │
|
||||
▼ ▼
|
||||
┌──────────────┐ ┌──────────────┐
|
||||
│ HIBP Check │ │ Reporter │
|
||||
│ (optional) │ │ term/json/ │
|
||||
│ client.go │ │ sarif │
|
||||
└──────┬───────┘ └──────────────┘
|
||||
│ ▲
|
||||
└─────────────────┘
|
||||
```
|
||||
|
||||
## Component Breakdown
|
||||
|
||||
### CLI Layer (`internal/cli/`)
|
||||
|
||||
**Purpose:** Parse command line arguments and orchestrate the scan workflow.
|
||||
|
||||
**Responsibilities:**
|
||||
- Route `scan`, `git`, `init`, `pyproject`, `config` commands to their handlers
|
||||
- Merge CLI flags with TOML config file values (CLI flags take precedence)
|
||||
- Create the appropriate Source, run the Pipeline, optionally check HIBP, produce output
|
||||
|
||||
**Interfaces:** Uses Cobra for argument parsing. Each command is a `cobra.Command` with a `RunE` function. The `executeScan` function in `scan.go` is shared between `scan` and `git` commands.
|
||||
|
||||
### Config Loader (`internal/config/`)
|
||||
|
||||
**Purpose:** Load and merge configuration from `.portia.toml` files.
|
||||
|
||||
**Responsibilities:**
|
||||
- Search for config files in three locations: current directory, `.portia/config.toml`, `~/.config/portia/config.toml`
|
||||
- Fall back to `pyproject.toml` (`[tool.portia]` table) when no `.portia.toml` is found
|
||||
- Parse TOML into a `Config` struct with sections for Rules, Scan, Output, HIBP, Allowlist
|
||||
- Provide default templates for `portia init` and `portia pyproject`
|
||||
|
||||
**Interfaces:** `Load(path string) (*Config, error)` returns a config or error. Empty path triggers auto-discovery.
|
||||
|
||||
### Source Interface (`internal/source/`)
|
||||
|
||||
**Purpose:** Produce chunks of text from various inputs (directories, git history).
|
||||
|
||||
**Responsibilities:**
|
||||
- Walk filesystem or git object tree
|
||||
- Skip binary files, excluded paths, oversized files
|
||||
- Split content into 50-line chunks with file path and line number metadata
|
||||
- Send chunks into a channel for pipeline consumption
|
||||
|
||||
**Interfaces:**
|
||||
```go
|
||||
type Source interface {
|
||||
Chunks(ctx context.Context, out chan<- types.Chunk) error
|
||||
String() string
|
||||
}
|
||||
```
|
||||
|
||||
**Directory source** (`directory.go`):
|
||||
- Uses `filepath.WalkDir` for filesystem traversal
|
||||
- Skips `.git`, `node_modules`, `vendor`, `__pycache__`, `.venv`
|
||||
- Checks file size against configurable max (default 1MB)
|
||||
- Chunks files into 50-line segments using a buffered scanner
|
||||
|
||||
**Git source** (`git.go`):
|
||||
- Uses go-git v5 for in-process git operations
|
||||
- `scanHistory`: walks commit log backwards, extracts file content from each commit's tree
|
||||
- `scanStaged`: reads git index entries for staged-only scanning
|
||||
- Supports `--branch`, `--since`, `--depth` filters
|
||||
|
||||
### Rule Registry (`internal/rules/`)
|
||||
|
||||
**Purpose:** Store detection rules and provide fast keyword-based lookup.
|
||||
|
||||
**Responsibilities:**
|
||||
- Store rules in a map keyed by rule ID
|
||||
- Provide `MatchKeywords(content)` that returns only rules whose keywords appear in the content
|
||||
- Support enabling/disabling rules
|
||||
- Maintain global path and value allowlists
|
||||
|
||||
**Interfaces:** `Register(rule)`, `Get(id)`, `All()`, `MatchKeywords(content)`, `Disable(ids...)`, `Len()`
|
||||
|
||||
### Detection Engine (`internal/engine/`)
|
||||
|
||||
**Purpose:** Apply rules to chunks and produce findings.
|
||||
|
||||
**Detector** (`detector.go`):
|
||||
- Takes a chunk, runs keyword pre-filter via registry
|
||||
- For each matched rule, scans line by line with regex
|
||||
- Extracts secret from capture group
|
||||
- Validates entropy if the rule has an entropy threshold
|
||||
- Runs through FilterFinding for false positive reduction
|
||||
|
||||
**Filter** (`filter.go`):
|
||||
- `IsPlaceholder` - checks against GlobalValueAllowlist patterns
|
||||
- `IsTemplated` - checks for `${...}`, `{{...}}`, `os.getenv()`, `process.env.`
|
||||
- `IsStopword` - splits secret on `_-./` delimiters, checks parts against 700+ stopwords
|
||||
- `IsAllowedPath` - checks file path against GlobalPathAllowlist
|
||||
- `FilterFinding` - orchestrates all checks, returns true if finding is real
|
||||
|
||||
**Pipeline** (`pipeline.go`):
|
||||
- Creates errgroup with source goroutine + N worker goroutines + collector goroutine
|
||||
- Workers pull chunks from channel, run detector, push findings to findings channel
|
||||
- Collector merges all findings, deduplicates by ruleID+filePath+secret+commitSHA
|
||||
|
||||
### HIBP Client (`internal/hibp/`)
|
||||
|
||||
**Purpose:** Check detected secrets against the Have I Been Pwned breach database.
|
||||
|
||||
**Responsibilities:**
|
||||
- SHA-1 hash computation
|
||||
- k-anonymity API queries (5-char prefix)
|
||||
- LRU cache (10,000 entries) for repeated lookups
|
||||
- Circuit breaker (5 failures = 60s cooldown)
|
||||
**Interfaces:** `Check(ctx, secret) (Result, error)`
|
||||
|
||||
### Reporters (`internal/reporter/`)
|
||||
|
||||
**Purpose:** Format scan results for output.
|
||||
|
||||
**Terminal** (`terminal.go`): Colored output with severity-based colors (red for CRITICAL, yellow for MEDIUM), secret masking (show first/last few characters), SHA truncation for git commits, HIBP breach status.
|
||||
|
||||
**JSON** (`json.go`): Structured JSON with `findings` array and `summary` object. Secrets are masked in output.
|
||||
|
||||
**SARIF** (`sarif.go`): SARIF v2.1.0 compliant output with tool metadata, rule definitions, results with locations and properties.
|
||||
|
||||
**Interfaces:** `Reporter` interface with `Report(w io.Writer, result *types.ScanResult) error`. Factory function `New(format) Reporter` returns the appropriate implementation.
|
||||
|
||||
## Data Flow
|
||||
|
||||
### Tracing: `portia scan ./myproject`
|
||||
|
||||
Step-by-step walkthrough of what happens when you run a directory scan:
|
||||
|
||||
```
|
||||
1. CLI parses arguments
|
||||
root.go:init() → cobra.OnInitialize(initConfig)
|
||||
scan.go:runScan() receives path="./myproject"
|
||||
|
||||
2. Config loading
|
||||
root.go:initConfig() → config.Load(cfgFile)
|
||||
Merges CLI flags with TOML config
|
||||
Format defaults to "terminal", maxSize defaults to 1MB
|
||||
|
||||
3. Registry setup
|
||||
scan.go:runScan() → rules.NewRegistry() + rules.RegisterBuiltins(reg)
|
||||
Loads 150 rules into the registry map
|
||||
Applies disabled rules from config: reg.Disable(cfg.Rules.Disable...)
|
||||
|
||||
4. Source creation
|
||||
scan.go:runScan() → source.NewDirectory(path, maxSize, excludes)
|
||||
Creates Directory struct with path, max file size, exclude patterns
|
||||
|
||||
5. Pipeline execution
|
||||
scan.go:executeScan() → engine.NewPipeline(reg).Run(ctx, src)
|
||||
|
||||
5a. Source goroutine starts
|
||||
Calls src.Chunks(ctx, chunks)
|
||||
WalkDir traverses ./myproject
|
||||
Skips .git, node_modules, vendor, binary extensions
|
||||
Splits each file into 50-line chunks
|
||||
Sends each chunk into the chunks channel
|
||||
|
||||
5b. Worker goroutines start (2-16 based on NumCPU)
|
||||
Each pulls chunks from the channel
|
||||
Calls detector.Detect(chunk):
|
||||
- reg.MatchKeywords(chunk.Content) → only rules with matching keywords
|
||||
- For each matched rule, scan each line with rule.Pattern regex
|
||||
- Extract secret from capture group
|
||||
- If rule has entropy threshold, compute Shannon entropy and compare
|
||||
- Run FilterFinding: IsPlaceholder → IsTemplated → IsStopword → path allowlist
|
||||
- If all checks pass, create Finding and send to findings channel
|
||||
|
||||
5c. Collector goroutine
|
||||
Pulls findings from findings channel
|
||||
Appends to allFindings slice (mutex-protected)
|
||||
|
||||
5d. Wait for all goroutines (errgroup.Wait)
|
||||
Dedup findings by ruleID+filePath+secret+commitSHA
|
||||
|
||||
6. HIBP verification (if --hibp flag)
|
||||
scan.go:checkHIBP(ctx, result)
|
||||
For each finding, calls client.Check(ctx, finding.Secret)
|
||||
Updates finding.HIBPStatus and finding.BreachCount
|
||||
|
||||
7. Reporter output
|
||||
scan.go:executeScan() → reporter.New(format).Report(os.Stdout, result)
|
||||
Terminal: colored table with severity, rule, file:line, masked secret
|
||||
JSON: structured JSON to stdout
|
||||
SARIF: SARIF v2.1.0 JSON to stdout
|
||||
```
|
||||
|
||||
## Concurrency Model
|
||||
|
||||
The pipeline uses Go's errgroup pattern for structured concurrency:
|
||||
|
||||
```
|
||||
errgroup
|
||||
┌────────────────────────────────┐
|
||||
│ │
|
||||
Source goroutine │ ──chunks──▶ Worker 1 │
|
||||
│ Worker 2 │
|
||||
│ ... │
|
||||
│ Worker N │
|
||||
│ ──findings──▶ │
|
||||
│ Collector │
|
||||
│ │
|
||||
└────────────────────────────────┘
|
||||
```
|
||||
|
||||
**Why bounded workers?** CPU-bound regex matching doesn't benefit from unbounded parallelism. Too many goroutines competing for CPU time causes context switching overhead. The formula `min(max(NumCPU, 2), 16)` gives 2 workers on single-core machines and caps at 16 on large servers.
|
||||
|
||||
**Why errgroup?** It provides two things: (1) if any goroutine returns an error, the context is cancelled and all goroutines wind down cleanly, and (2) `g.Wait()` blocks until all goroutines complete, giving you a single point to check for errors.
|
||||
|
||||
**Channel sizing:** Channels are buffered at `workers * 4`. This allows the source to stay ahead of workers (avoiding blocking on sends) without unbounded memory growth. If workers are slow, the source will block once the buffer fills, providing natural backpressure.
|
||||
|
||||
**The detectWg dance:** Workers share a separate `sync.WaitGroup` so we know when all detection is done. The collector goroutine runs in the same errgroup but only closes after all workers finish. This prevents the collector from exiting early while findings are still being produced. See `pipeline.go:52-77`.
|
||||
|
||||
## Configuration Resolution
|
||||
|
||||
Configuration is resolved in this order (later overrides earlier):
|
||||
|
||||
```
|
||||
1. Defaults (hardcoded)
|
||||
Format: "terminal"
|
||||
MaxSize: 1MB (1 << 20)
|
||||
Workers: min(max(NumCPU, 2), 16)
|
||||
HIBP: disabled
|
||||
Verbose: false
|
||||
NoColor: false
|
||||
|
||||
2. Config file (.portia.toml)
|
||||
Searched in order:
|
||||
.portia.toml (current directory)
|
||||
.portia/config.toml
|
||||
~/.config/portia/config.toml
|
||||
First found is loaded. Later paths are not checked.
|
||||
If none found, falls back to pyproject.toml ([tool.portia] table).
|
||||
|
||||
3. CLI flags
|
||||
--format, --verbose, --no-color, --exclude, --max-size, --hibp, --config
|
||||
These always win over config file values.
|
||||
```
|
||||
|
||||
This merge logic is in `internal/cli/root.go:initConfig()`. The pattern is: check if the CLI flag was explicitly set (non-zero/non-empty), and only fall back to config file value if the flag wasn't set.
|
||||
|
||||
## Rule Matching Strategy
|
||||
|
||||
The detection pipeline is optimized for speed. Regex matching is expensive, so the goal is to avoid running regex against content that will never match.
|
||||
|
||||
```
|
||||
Content chunk (50 lines of code)
|
||||
│
|
||||
▼
|
||||
┌───────────────────┐
|
||||
│ Keyword Filter │ ← O(rules * keywords) string.Contains
|
||||
│ ~95% eliminated │
|
||||
└────────┬──────────┘
|
||||
│ Only rules whose keywords appear in this chunk
|
||||
▼
|
||||
┌───────────────────┐
|
||||
│ Line-by-Line │ ← O(lines * matched_rules) regex
|
||||
│ Regex Matching │
|
||||
└────────┬──────────┘
|
||||
│ Raw matches with capture groups
|
||||
▼
|
||||
┌───────────────────┐
|
||||
│ Secret Extract │ ← Extract from capture group
|
||||
│ + Entropy Check │ Discard if below threshold
|
||||
└────────┬──────────┘
|
||||
│ Validated candidates
|
||||
▼
|
||||
┌───────────────────┐
|
||||
│ Filter Chain │ ← IsPlaceholder → IsTemplated
|
||||
│ 5-layer check │ → IsStopword → Allowlists
|
||||
└────────┬──────────┘
|
||||
│ Real findings only
|
||||
▼
|
||||
Finding
|
||||
```
|
||||
|
||||
The keyword filter is the key performance optimization. If a 50-line chunk of HTML doesn't contain any strings like `password`, `secret`, `key`, `token`, `AKIA`, `ghp_`, `sk_live`, etc., then zero rules will match and zero regex patterns need to run against it. In practice, this eliminates the vast majority of chunks.
|
||||
|
|
@ -0,0 +1,308 @@
|
|||
# Implementation Walkthrough
|
||||
|
||||
This document walks through the key code files in Portia. For each section, we'll look at what the code does, why it's designed that way, and what to watch out for.
|
||||
|
||||
## Shannon Entropy (`internal/rules/entropy.go`)
|
||||
|
||||
The entropy module answers one question: "Is this string random enough to be a secret?"
|
||||
|
||||
**ShannonEntropy function:**
|
||||
The function takes a string and a charset (the set of all possible characters). It counts how many times each character appears, computes the probability of each character (count / total), then sums `-p * log₂(p)` across all characters.
|
||||
|
||||
The charset parameter matters for the calculation. If you compute entropy using only the characters present in the string, every string with all unique characters gets the same entropy. By computing against the full charset (e.g., all 62 alphanumeric characters), the result reflects how much of the available randomness space the string actually uses.
|
||||
|
||||
**DetectCharset function:**
|
||||
Before computing entropy, Portia guesses the charset:
|
||||
- If all characters are `0-9a-f`, it's hex. Use the 16-character hex charset.
|
||||
- If all characters are `A-Za-z0-9+/=`, it's base64. Use the 64-character base64 charset.
|
||||
- Otherwise, use the 62-character alphanumeric charset as default.
|
||||
|
||||
This matters because a hex string `deadbeefcafe` has different entropy depending on whether you evaluate it against 16 possible characters (hex) or 62 (alphanumeric). The hex evaluation is more generous, correctly identifying it as moderately random within its charset.
|
||||
|
||||
**Thresholds in practice:**
|
||||
- `password = "admin"` → entropy ~2.3 (below most thresholds, filtered out)
|
||||
- `password = "xK9mP2vL5nQ8jR3t"` → entropy ~4.0 (above threshold, flagged)
|
||||
- `AKIAIOSFODNN7EXAMPLE` → no entropy check needed (structural rule, AKIA prefix is sufficient)
|
||||
|
||||
## Rule Registry (`internal/rules/registry.go`)
|
||||
|
||||
The registry is a simple map from rule ID to rule struct, with a few key methods:
|
||||
|
||||
**MatchKeywords** is the performance-critical function. For each rule in the registry, it checks if any of the rule's keywords appear (case-insensitive) in the content string. This is O(rules * keywords * content_length) in the worst case, but in practice, `strings.Contains` with short keywords against medium-length chunks is fast.
|
||||
|
||||
The return value is a slice of matching rules. If a chunk contains `password`, the registry returns all rules that have `password` as a keyword. If the chunk contains `AKIA`, it returns the AWS access key rules.
|
||||
|
||||
**Global allowlists** are defined at the bottom of `registry.go`:
|
||||
- `GlobalPathAllowlist` - regex patterns for paths to skip (go.mod, package-lock.json, node_modules/, vendor/, binary extensions, minified JS)
|
||||
- `GlobalValueAllowlist` - regex patterns for values to ignore (example, test, dummy, fake, placeholder, YOUR_API_KEY, xxxx..., TODO, CHANGEME)
|
||||
|
||||
These are separate from per-rule allowlists. A per-rule allowlist (like AWS's `AKIAIOSFODNN7EXAMPLE`) only applies to that specific rule. Global allowlists apply to all rules.
|
||||
|
||||
## Detection Rules (`internal/rules/builtin.go`)
|
||||
|
||||
Each rule is a `types.Rule` struct with these fields:
|
||||
|
||||
```go
|
||||
type Rule struct {
|
||||
ID string // unique identifier like "aws-access-key-id"
|
||||
Description string // human-readable "AWS Access Key ID"
|
||||
Severity Severity // SeverityCritical, SeverityHigh, etc.
|
||||
Keywords []string // fast pre-filter: ["AKIA", "ABIA", "ACCA", "ASIA"]
|
||||
Pattern *regexp.Regexp // the actual detection regex
|
||||
SecretGroup int // which capture group contains the secret (0=whole match)
|
||||
Entropy *float64 // minimum entropy threshold (nil = no check)
|
||||
Allowlist Allowlist // per-rule path/value/stopword overrides
|
||||
SecretType SecretType // classification: APIKey, Token, Password, etc.
|
||||
}
|
||||
```
|
||||
|
||||
**Walking through the AWS access key rule:**
|
||||
|
||||
```go
|
||||
{
|
||||
ID: "aws-access-key-id",
|
||||
Description: "AWS Access Key ID",
|
||||
Severity: types.SeverityCritical,
|
||||
Keywords: []string{"AKIA", "ABIA", "ACCA", "ASIA"},
|
||||
Pattern: regexp.MustCompile(`\b((?:AKIA|ABIA|ACCA|ASIA)[0-9A-Z]{16})\b`),
|
||||
SecretGroup: 1,
|
||||
SecretType: types.SecretTypeAPIKey,
|
||||
}
|
||||
```
|
||||
|
||||
- **Keywords**: Four possible prefixes. If the chunk doesn't contain any of these four strings, skip this rule entirely.
|
||||
- **Pattern**: Word boundary `\b`, then one of four prefixes, then exactly 16 uppercase alphanumeric characters, then word boundary. The whole match is captured in group 1.
|
||||
- **SecretGroup**: 1 means extract from the first parenthesized group (the entire key).
|
||||
- **No entropy threshold**: AWS keys have a fixed structure, so entropy validation isn't needed. The prefix + length is sufficient.
|
||||
- **No allowlist**: The global value allowlist already catches `AKIAIOSFODNN7EXAMPLE`.
|
||||
|
||||
**Walking through the generic password rule:**
|
||||
|
||||
```go
|
||||
{
|
||||
ID: "generic-password",
|
||||
Description: "Password in Assignment",
|
||||
Severity: types.SeverityHigh,
|
||||
Keywords: []string{"password", "passwd", "pwd"},
|
||||
Pattern: regexp.MustCompile(`(?i)(?:password|passwd|pwd)\s*[:=]\s*['"]([^'"]{8,})['"]`),
|
||||
SecretGroup: 1,
|
||||
Entropy: ptr(3.5),
|
||||
SecretType: types.SecretTypePassword,
|
||||
}
|
||||
```
|
||||
|
||||
- **Keywords**: Three variations of "password"
|
||||
- **Pattern**: Case-insensitive match for password/passwd/pwd, followed by `:` or `=`, optional whitespace, then a quoted string of at least 8 characters. Group 1 captures just the password value.
|
||||
- **Entropy threshold**: 3.5 bits. This filters out `password = "admin123"` (low entropy) while catching `password = "xK9mP2vL5nQ8jR3t"` (high entropy).
|
||||
- The `ptr()` helper function creates a `*float64` from a literal, since Go doesn't allow taking the address of a constant directly.
|
||||
|
||||
## Directory Source (`internal/source/directory.go`)
|
||||
|
||||
The directory source walks a filesystem and produces chunks:
|
||||
|
||||
**WalkDir callback**: For each file system entry:
|
||||
1. Check context cancellation (allows clean shutdown)
|
||||
2. Skip known non-interesting directories (`.git`, `node_modules`, `vendor`, `__pycache__`, `.venv`)
|
||||
3. Check if the relative path matches any exclude patterns
|
||||
4. Skip binary file extensions (`.png`, `.jpg`, `.exe`, `.zip`, etc.)
|
||||
5. Check file size against the max limit (default 1MB)
|
||||
6. If all checks pass, call `emitChunks`
|
||||
|
||||
**emitChunks**: Opens the file and reads it line by line using `bufio.Scanner`:
|
||||
- Accumulates lines into a `strings.Builder`
|
||||
- Every 50 lines, sends the accumulated text as a `types.Chunk` with the file's relative path and the starting line number
|
||||
- After the loop, sends any remaining lines as a final chunk
|
||||
|
||||
**Why 50-line chunks?** This is a tradeoff between memory usage and detection accuracy. Larger chunks use more memory per worker. Smaller chunks might split a multi-line secret across two chunks. 50 lines is a practical middle ground: most secrets fit on a single line, and 50 lines is small enough to process quickly.
|
||||
|
||||
**isExcluded function**: Checks two patterns:
|
||||
- `filepath.Match` against the filename (base name only). This handles patterns like `*.env`
|
||||
- `strings.Contains` against the full relative path. This handles patterns like `test/fixtures`
|
||||
|
||||
## Git Source (`internal/source/git.go`)
|
||||
|
||||
The git source uses go-git v5 to scan repository history without shelling out to the `git` binary.
|
||||
|
||||
**scanHistory**: Opens the repository with `git.PlainOpen`, then:
|
||||
1. Gets a commit iterator filtered by branch (if specified)
|
||||
2. For each commit, checks if the date is after `--since` and within `--depth`
|
||||
3. Gets the commit's tree and walks all entries
|
||||
4. For each blob (file), reads the content, checks size/excludes/binary extensions
|
||||
5. Splits into 50-line chunks with commit metadata (SHA, author, date)
|
||||
|
||||
**scanStaged**: For pre-commit scanning:
|
||||
1. Opens the repository and reads the git index (staging area)
|
||||
2. For each index entry, reads the blob content from the object store
|
||||
3. Produces chunks for only the files that are currently staged
|
||||
|
||||
**readBlob**: Reads a git blob object into a string:
|
||||
```go
|
||||
func readBlob(obj *object.Blob) (string, error) {
|
||||
reader, err := obj.Reader()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer reader.Close()
|
||||
data, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(data), nil
|
||||
}
|
||||
```
|
||||
|
||||
This uses `io.ReadAll` rather than `strings.Builder.ReadFrom` because `strings.Builder` doesn't have a `ReadFrom` method. A common mistake when working with go-git blobs.
|
||||
|
||||
## Detector (`internal/engine/detector.go`)
|
||||
|
||||
The detector is where rules meet content:
|
||||
|
||||
**Detect function flow:**
|
||||
1. Call `registry.MatchKeywords(chunk.Content)` to get only relevant rules
|
||||
2. If no rules match keywords, return nil immediately (fast path)
|
||||
3. Split chunk content into lines
|
||||
4. For each matched rule, for each line:
|
||||
- Run `rule.Pattern.FindAllStringSubmatchIndex(line, -1)` to find all matches
|
||||
- For each match, call `extractSecret` to get the secret from the capture group
|
||||
- If the rule has an entropy threshold, compute entropy and skip if below threshold
|
||||
- Call `FilterFinding` for false positive checks
|
||||
- If all checks pass, append to findings
|
||||
|
||||
**extractSecret function:**
|
||||
```go
|
||||
func extractSecret(line string, loc []int, group int) string {
|
||||
if group > 0 && len(loc) > group*2+1 {
|
||||
start := loc[group*2]
|
||||
end := loc[group*2+1]
|
||||
if start >= 0 && end >= 0 {
|
||||
return line[start:end]
|
||||
}
|
||||
}
|
||||
if len(loc) >= 2 {
|
||||
return line[loc[0]:loc[1]]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
```
|
||||
|
||||
The `loc` array from `FindAllStringSubmatchIndex` contains pairs of start/end indices for each capture group. Group 0 is the whole match (indices 0,1), group 1 is the first parenthesized group (indices 2,3), etc. If the requested group doesn't exist or has negative indices (meaning the group didn't participate in the match), fall back to the whole match.
|
||||
|
||||
## Filter (`internal/engine/filter.go`)
|
||||
|
||||
The filter chain is the final defense against false positives:
|
||||
|
||||
**IsStopword**: The critical fix here was changing from substring matching to delimiter-split exact matching. The original implementation checked if any stopword was a substring of the secret. This caused `AKIAIOSFODNN7EXAMPLE` to match because "example" appeared as a substring. The fix:
|
||||
|
||||
```go
|
||||
parts := strings.FieldsFunc(lower, func(r rune) bool {
|
||||
return r == '_' || r == '-' || r == '.' || r == '/'
|
||||
})
|
||||
for _, part := range parts {
|
||||
if _, ok := stopwords[part]; ok {
|
||||
return true
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
This splits on common delimiter characters and checks each part independently. `AKIAIOSFODNN7EXAMPLE` doesn't split into "example" because there's no delimiter before it. But `module_controller_config` splits into ["module", "controller", "config"], all of which are stopwords.
|
||||
|
||||
**FilterFinding orchestration:**
|
||||
```
|
||||
IsPlaceholder(secret) → true = skip
|
||||
IsTemplated(secret) → true = skip
|
||||
IsStopword(secret) → true = skip
|
||||
rule.Allowlist.Values → match = skip
|
||||
GlobalPathAllowlist → match = skip
|
||||
rule.Allowlist.Paths → match = skip
|
||||
All checks pass → finding is real
|
||||
```
|
||||
|
||||
Each layer is independent. A finding only needs to be caught by one layer to be filtered out.
|
||||
|
||||
## Pipeline (`internal/engine/pipeline.go`)
|
||||
|
||||
The pipeline connects everything with Go concurrency primitives:
|
||||
|
||||
**Setup:**
|
||||
```go
|
||||
chunks := make(chan types.Chunk, p.workers*4)
|
||||
findingsCh := make(chan types.Finding, p.workers*4)
|
||||
g, gctx := errgroup.WithContext(ctx)
|
||||
```
|
||||
|
||||
Two buffered channels: one for chunks (source → workers), one for findings (workers → collector). Buffer size is `workers * 4` for backpressure balance.
|
||||
|
||||
**Source goroutine:** Runs `src.Chunks(gctx, chunks)` and defers `close(chunks)`. When the source finishes (or context is cancelled), the channel closes and workers drain remaining items.
|
||||
|
||||
**Worker goroutines:** Each worker loops over the chunks channel:
|
||||
```go
|
||||
for chunk := range chunks {
|
||||
if gctx.Err() != nil {
|
||||
return gctx.Err()
|
||||
}
|
||||
results := p.detector.Detect(chunk)
|
||||
for _, f := range results {
|
||||
findingsCh <- f
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Workers share a `sync.WaitGroup` separate from the errgroup. When all workers are done, a goroutine closes the findings channel.
|
||||
|
||||
**Collector goroutine:** Simple loop that collects all findings into a slice:
|
||||
```go
|
||||
for f := range findingsCh {
|
||||
mu.Lock()
|
||||
allFindings = append(allFindings, f)
|
||||
mu.Unlock()
|
||||
}
|
||||
```
|
||||
|
||||
The mutex isn't strictly necessary since there's only one collector goroutine, but it protects against future changes and makes the data race detector happy.
|
||||
|
||||
**Deduplication:** After all goroutines complete, `dedup` removes duplicate findings by creating a composite key from `ruleID + "|" + filePath + "|" + secret + "|" + commitSHA`. This handles cases where the same secret appears in overlapping chunks or across multiple git commits.
|
||||
|
||||
## HIBP Client (`internal/hibp/client.go`)
|
||||
|
||||
The HIBP client checks secrets against Troy Hunt's breach database:
|
||||
|
||||
**SHA-1 hashing:** The secret is hashed with SHA-1 (yes, SHA-1 is cryptographically broken, but HIBP uses it as a lookup key, not for security). The hash is uppercased and split: first 5 characters are the prefix, remaining 35 are the suffix.
|
||||
|
||||
**k-anonymity query:** Send the prefix to `https://api.pwnedpasswords.com/range/{prefix}`. The API returns all hash suffixes that share that prefix, along with occurrence counts:
|
||||
```
|
||||
0018A45C4D1DEF81644B54AB7F969B88D65:21
|
||||
00D4F6E8FA6EECAD2A3AA415EEC418D38EC:2
|
||||
```
|
||||
|
||||
The client parses each line, splits on `:`, and checks if any suffix matches ours. If so, the secret was found in a breach.
|
||||
|
||||
**LRU cache:** Before making an API call, check the cache using the 5-character prefix as the key. The LRU cache holds 10,000 entries. Since each prefix covers all secrets with that prefix, caching is very effective when scanning large codebases with similar secrets.
|
||||
|
||||
**Circuit breaker:** Wraps the HTTP call in a `gobreaker.CircuitBreaker`. Settings:
|
||||
- Max consecutive failures: 5
|
||||
- Timeout (recovery period): 60 seconds
|
||||
- After 5 straight failures, the circuit opens and immediately returns an error for all subsequent calls. After 60 seconds, it enters half-open state and lets one request through to test if the API is back.
|
||||
|
||||
**baseURL field:** The client has a `baseURL` field that defaults to the real HIBP API URL. In tests, this is overridden to point at an `httptest.Server`. This avoids the need for interface-based mocking and keeps the code simple.
|
||||
|
||||
## Reporters (`internal/reporter/`)
|
||||
|
||||
**Terminal reporter** (`terminal.go`):
|
||||
- Sorts findings by severity (CRITICAL first)
|
||||
- Colors: red for CRITICAL, red (non-bold) for HIGH, yellow for MEDIUM, cyan for LOW
|
||||
- Masks secrets: shows first 4-6 and last 4-6 characters with asterisks in between
|
||||
- Truncates commit SHAs to 8 characters for readability
|
||||
- Shows entropy values when present
|
||||
- Shows HIBP breach status and count when checked
|
||||
|
||||
**JSON reporter** (`json.go`):
|
||||
- Produces a JSON object with `findings` array and `summary` object
|
||||
- Each finding has: rule_id, description, severity, secret (masked), entropy, file, line, commit, author, hibp_status, breach_count
|
||||
- Summary has: total_findings, total_rules, duration, hibp_checked, hibp_breached
|
||||
|
||||
**SARIF reporter** (`sarif.go`):
|
||||
- Produces SARIF v2.1.0 compliant JSON
|
||||
- Maps Portia severity to SARIF levels: CRITICAL/HIGH = "error", MEDIUM = "warning", LOW = "note"
|
||||
- Each finding becomes a SARIF `result` with `ruleId`, `message`, `level`, and `locations`
|
||||
- Rule definitions are included in the `tool.driver.rules` array
|
||||
- Custom properties (entropy, HIBP status, masked secret) go in `result.properties`
|
||||
|
|
@ -0,0 +1,207 @@
|
|||
# Extension Challenges
|
||||
|
||||
These challenges extend Portia beyond its current capabilities. Each one teaches a different skill. They're ordered roughly by difficulty and build on the existing codebase without requiring major refactors.
|
||||
|
||||
## Challenge 1: Pre-commit Hook Integration
|
||||
|
||||
**Difficulty:** Easy | **Time:** 1-2 hours | **Teaches:** Git hooks, shell scripting, developer workflow
|
||||
|
||||
Write a script that installs Portia as a Git pre-commit hook. When a developer runs `git commit`, the hook should:
|
||||
|
||||
1. Build Portia (or use a pre-built binary)
|
||||
2. Run `portia git --staged` to scan only staged files
|
||||
3. If secrets are found, print the findings and abort the commit
|
||||
4. If no secrets are found, allow the commit to proceed
|
||||
|
||||
**Starting point:** Create a `scripts/install-hook.sh` that writes a pre-commit hook to `.git/hooks/pre-commit`. The hook script should call `portia git --staged --format terminal` and check the exit code.
|
||||
|
||||
**Hints:**
|
||||
- Git hooks must be executable (`chmod +x`)
|
||||
- The hook should exit 0 to allow the commit, non-zero to abort
|
||||
- You'll need to add exit code support to Portia's CLI (currently it always exits 0). Add `--exit-code` flag that returns exit code 1 when secrets are found. Modify `executeScan` in `internal/cli/scan.go` to call `os.Exit(1)` when findings exist and the flag is set.
|
||||
- Consider adding a `--quiet` flag that suppresses the banner and spinner for hook usage
|
||||
|
||||
**Bonus:** Make it work with the `pre-commit` framework (https://pre-commit.com) by creating a `.pre-commit-hooks.yaml` in the repo root.
|
||||
|
||||
## Challenge 2: Custom Rule YAML Loader
|
||||
|
||||
**Difficulty:** Medium | **Time:** 2-3 hours | **Teaches:** YAML/TOML parsing, rule validation, config extensibility
|
||||
|
||||
Add support for user-defined detection rules in a YAML or TOML file. Users should be able to create `.portia/rules.yml`:
|
||||
|
||||
```yaml
|
||||
rules:
|
||||
- id: "internal-api-key"
|
||||
description: "Internal API key format"
|
||||
severity: HIGH
|
||||
keywords: ["ikey_"]
|
||||
pattern: 'ikey_[a-zA-Z0-9]{32}'
|
||||
secret_group: 0
|
||||
entropy: 3.5
|
||||
```
|
||||
|
||||
**Starting point:** Create `internal/rules/custom.go` with a `LoadCustomRules(path string) ([]*types.Rule, error)` function. Call this from `scan.go` after registering builtins.
|
||||
|
||||
**Hints:**
|
||||
- Use `gopkg.in/yaml.v3` for YAML parsing
|
||||
- Validate the regex pattern by calling `regexp.Compile` and returning a clear error if it fails
|
||||
- Validate severity against allowed values
|
||||
- Check for duplicate rule IDs against the existing registry
|
||||
- Consider supporting `allowlist` in the YAML format with path and value patterns
|
||||
|
||||
**Gotcha:** Keywords are critical for performance. If a custom rule has no keywords, it'll run its regex against every chunk. Either require at least one keyword or warn the user that empty keywords will be slow.
|
||||
|
||||
## Challenge 3: Incremental Scanning with Cache
|
||||
|
||||
**Difficulty:** Medium | **Time:** 2-3 hours | **Teaches:** Hashing, file-based caching, performance optimization
|
||||
|
||||
Add a `.portia-cache/scan.json` file that stores SHA-256 hashes of previously scanned files. On subsequent scans, skip files whose hash hasn't changed.
|
||||
|
||||
**Starting point:** Create `internal/cache/scan.go` with:
|
||||
- `type ScanCache struct` holding a map from relative file path to file hash
|
||||
- `Load(path) (*ScanCache, error)` and `Save(path) error` for persistence
|
||||
- `IsChanged(relPath string, content []byte) bool` that computes SHA-256 and compares
|
||||
|
||||
**Hints:**
|
||||
- Store the cache in `.portia-cache/scan.json` in the scanned directory
|
||||
- Use `crypto/sha256` for hashing
|
||||
- The cache should include the rule count as metadata. If rules change (new rule added), invalidate the entire cache.
|
||||
- Add a `--no-cache` flag to force a full rescan
|
||||
- Add cache invalidation in `internal/cli/scan.go` before creating the source
|
||||
- Consider adding the Portia version to the cache metadata so version upgrades invalidate the cache
|
||||
|
||||
**Performance impact:** On a 10,000-file codebase where only 50 files changed, this reduces scan time by ~99.5%.
|
||||
|
||||
## Challenge 4: Git Blame Integration
|
||||
|
||||
**Difficulty:** Medium | **Time:** 3-4 hours | **Teaches:** Git blame API, attribution, enriched output
|
||||
|
||||
After detecting a secret, run `git blame` on the file to determine who committed it and when. Add this information to the finding.
|
||||
|
||||
**Starting point:** The `types.Finding` struct already has `Author` and `CommitDate` fields, but they're only populated during git history scans. For directory scans, these fields are empty.
|
||||
|
||||
**Hints:**
|
||||
- Use go-git's `git.Blame` function: `blame, err := git.BlameCommit(commit, path)`
|
||||
- The blame result gives you the commit SHA, author, and date for each line
|
||||
- Match the finding's `LineNumber` to the blame result to get attribution
|
||||
- This should be opt-in (`--blame` flag) since it adds overhead
|
||||
- For files outside a git repo, skip blame silently
|
||||
- Add blame data to all three reporter formats (terminal, JSON, SARIF)
|
||||
|
||||
**Gotcha:** `git.Blame` requires walking the full commit history for the file. On large repos, this can be slow. Consider caching blame results per file.
|
||||
|
||||
## Challenge 5: Multi-repo Scanning
|
||||
|
||||
**Difficulty:** Medium | **Time:** 3-4 hours | **Teaches:** Configuration management, concurrent I/O, aggregation
|
||||
|
||||
Add a `portia scan-all` command that reads a config file listing multiple repositories and scans them all, producing a unified report.
|
||||
|
||||
```toml
|
||||
[[repos]]
|
||||
path = "/home/dev/api-server"
|
||||
excludes = ["vendor/"]
|
||||
|
||||
[[repos]]
|
||||
path = "/home/dev/frontend"
|
||||
excludes = ["node_modules/", "dist/"]
|
||||
|
||||
[[repos]]
|
||||
url = "https://github.com/org/service.git"
|
||||
branch = "main"
|
||||
depth = 50
|
||||
```
|
||||
|
||||
**Starting point:** Create `internal/cli/scanall.go` with a new cobra command.
|
||||
|
||||
**Hints:**
|
||||
- Parse the config file with `pelletier/go-toml`
|
||||
- For `url` entries, clone to a temp directory using `git.PlainClone`
|
||||
- Run each repo scan concurrently using an errgroup
|
||||
- Prefix each finding's `FilePath` with the repo name/path for disambiguation
|
||||
- Consider a `--parallel N` flag to control concurrency
|
||||
- Clean up cloned temp directories on exit (use `defer`)
|
||||
|
||||
## Challenge 6: GitHub Action
|
||||
|
||||
**Difficulty:** Hard | **Time:** 4-6 hours | **Teaches:** GitHub Actions, Docker, SARIF integration, CI/CD
|
||||
|
||||
Build a GitHub Action that runs Portia on pull requests and uploads results to GitHub Code Scanning.
|
||||
|
||||
**Starting point:** Create `.github/action/action.yml` and a Dockerfile.
|
||||
|
||||
**Structure:**
|
||||
```
|
||||
.github/action/
|
||||
├── action.yml # Action metadata
|
||||
├── Dockerfile # Build Portia in a container
|
||||
└── entrypoint.sh # Run Portia and upload SARIF
|
||||
```
|
||||
|
||||
**Hints:**
|
||||
- The `action.yml` should accept inputs: `path` (default `.`), `format` (default `sarif`), `exclude` (optional), `hibp` (default false)
|
||||
- The Dockerfile should be a multi-stage build: compile Portia in a Go image, copy the binary to a slim runtime image
|
||||
- `entrypoint.sh` runs `portia scan --format sarif $INPUT_PATH > results.sarif`, then uploads using `gh api repos/{owner}/{repo}/code-scanning/sarifs`
|
||||
- Use `github.sha` for the commit SHA in the SARIF upload
|
||||
- The Action should fail (exit 1) if CRITICAL or HIGH findings are detected
|
||||
|
||||
**Testing:** Create a `.github/workflows/test-action.yml` that tests the action against `testdata/fixtures/`.
|
||||
|
||||
## Challenge 7: Secret Rotation Suggestions
|
||||
|
||||
**Difficulty:** Hard | **Time:** 4-6 hours | **Teaches:** Provider APIs, remediation guidance, structured output
|
||||
|
||||
After detecting a leaked secret, provide specific rotation instructions for each provider.
|
||||
|
||||
**Starting point:** Create `internal/remediation/remediation.go` with a map from rule ID to remediation steps.
|
||||
|
||||
**Example output:**
|
||||
```
|
||||
CRITICAL aws-access-key-id config.py:1
|
||||
|
||||
Rotation steps:
|
||||
1. Go to AWS IAM Console → Users → Security credentials
|
||||
2. Create a new access key
|
||||
3. Update all services using the old key
|
||||
4. Deactivate the old key (don't delete yet)
|
||||
5. After 24-48 hours with no issues, delete the old key
|
||||
6. Run: aws sts get-caller-identity (to verify new key works)
|
||||
|
||||
Documentation: https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html
|
||||
```
|
||||
|
||||
**Hints:**
|
||||
- Create a `Remediation` struct with `Steps []string`, `DocURL string`, `CLICommand string`
|
||||
- Map rule IDs to remediations: `aws-access-key-id` → AWS IAM rotation, `github-pat-classic` → GitHub settings rotation, `stripe-live-secret` → Stripe dashboard rotation
|
||||
- Add a `--remediate` flag to the CLI
|
||||
- For the terminal reporter, print remediation steps indented under each finding
|
||||
- For JSON/SARIF, include remediation in properties
|
||||
|
||||
## Challenge 8: Aho-Corasick Keyword Matching
|
||||
|
||||
**Difficulty:** Hard | **Time:** 3-4 hours | **Teaches:** Trie data structures, string matching algorithms, performance
|
||||
|
||||
Replace the linear keyword scan in `MatchKeywords` with an Aho-Corasick automaton for O(n) matching against all keywords simultaneously.
|
||||
|
||||
**Current approach** (`internal/rules/registry.go` MatchKeywords):
|
||||
For each rule, for each keyword, call `strings.Contains`. This is O(rules * keywords * content_length).
|
||||
|
||||
**Better approach:**
|
||||
Build a trie from all keywords at registry initialization time. At scan time, run the content through the automaton once. The automaton reports which keywords matched, and you map those back to rules.
|
||||
|
||||
**Starting point:** Use `github.com/cloudflare/ahocorasick` or implement your own.
|
||||
|
||||
**Hints:**
|
||||
- Build the automaton in `Registry.Register` or in a `Finalize()` method called after all rules are registered
|
||||
- The automaton should be case-insensitive (convert all keywords and content to lowercase)
|
||||
- Map each keyword back to its rule(s) using a reverse index
|
||||
- Benchmark before and after: `go test -bench=BenchmarkMatchKeywords -benchmem`
|
||||
- The improvement will be most noticeable on large files with many rules. On small files with few rules, the overhead of building the automaton might make it slower.
|
||||
|
||||
**Expected improvement:** On a 500-line file with 150 rules averaging 2 keywords each, the current approach does ~300 `strings.Contains` calls. Aho-Corasick does one pass through the content. For large codebases with thousands of files, this adds up.
|
||||
|
||||
## General Tips
|
||||
|
||||
- **Write tests first.** Every challenge should start with a failing test. The existing test patterns in `internal/engine/detector_test.go` and `internal/engine/integration_test.go` are good templates.
|
||||
- **Keep changes isolated.** Each challenge should be implementable without modifying the core detection logic. Use interfaces and composition to extend rather than modify.
|
||||
- **Benchmark when making performance claims.** Go's `testing.B` benchmarks are simple to write. If you claim something is faster, prove it with numbers.
|
||||
- **Check the Justfile.** Run `just ci` before considering any challenge complete. All existing tests should still pass.
|
||||
|
|
@ -0,0 +1,163 @@
|
|||
// ©AngelaMos | 2026
|
||||
// types.go
|
||||
|
||||
package types
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"time"
|
||||
)
|
||||
|
||||
type SecretType int
|
||||
|
||||
const (
|
||||
SecretTypePassword SecretType = iota
|
||||
SecretTypeAPIKey
|
||||
SecretTypeToken
|
||||
SecretTypePrivateKey
|
||||
SecretTypeConnectionString
|
||||
SecretTypeGenericHighEntropy
|
||||
)
|
||||
|
||||
func (s SecretType) String() string {
|
||||
switch s {
|
||||
case SecretTypePassword:
|
||||
return "password"
|
||||
case SecretTypeAPIKey:
|
||||
return "api-key"
|
||||
case SecretTypeToken:
|
||||
return "token"
|
||||
case SecretTypePrivateKey:
|
||||
return "private-key"
|
||||
case SecretTypeConnectionString:
|
||||
return "connection-string"
|
||||
case SecretTypeGenericHighEntropy:
|
||||
return "generic-high-entropy"
|
||||
default:
|
||||
return "unknown"
|
||||
}
|
||||
}
|
||||
|
||||
type Severity int
|
||||
|
||||
const (
|
||||
SeverityCritical Severity = iota
|
||||
SeverityHigh
|
||||
SeverityMedium
|
||||
SeverityLow
|
||||
)
|
||||
|
||||
func (s Severity) String() string {
|
||||
switch s {
|
||||
case SeverityCritical:
|
||||
return "CRITICAL"
|
||||
case SeverityHigh:
|
||||
return "HIGH"
|
||||
case SeverityMedium:
|
||||
return "MEDIUM"
|
||||
case SeverityLow:
|
||||
return "LOW"
|
||||
default:
|
||||
return "UNKNOWN"
|
||||
}
|
||||
}
|
||||
|
||||
func ParseSeverity(s string) Severity {
|
||||
switch s {
|
||||
case "CRITICAL", "critical":
|
||||
return SeverityCritical
|
||||
case "HIGH", "high":
|
||||
return SeverityHigh
|
||||
case "MEDIUM", "medium":
|
||||
return SeverityMedium
|
||||
case "LOW", "low":
|
||||
return SeverityLow
|
||||
default:
|
||||
return SeverityLow
|
||||
}
|
||||
}
|
||||
|
||||
func (s Severity) Rank() int {
|
||||
return int(s)
|
||||
}
|
||||
|
||||
type HIBPStatus int
|
||||
|
||||
const (
|
||||
HIBPUnchecked HIBPStatus = iota
|
||||
HIBPBreached
|
||||
HIBPClean
|
||||
HIBPSkipped
|
||||
HIBPError
|
||||
)
|
||||
|
||||
func (h HIBPStatus) String() string {
|
||||
switch h {
|
||||
case HIBPUnchecked:
|
||||
return "unchecked"
|
||||
case HIBPBreached:
|
||||
return "breached"
|
||||
case HIBPClean:
|
||||
return "clean"
|
||||
case HIBPSkipped:
|
||||
return "skipped"
|
||||
case HIBPError:
|
||||
return "error"
|
||||
default:
|
||||
return "unchecked"
|
||||
}
|
||||
}
|
||||
|
||||
type Allowlist struct {
|
||||
Paths []*regexp.Regexp
|
||||
Values []*regexp.Regexp
|
||||
Stopwords []string
|
||||
}
|
||||
|
||||
type Rule struct {
|
||||
ID string
|
||||
Description string
|
||||
Severity Severity
|
||||
Keywords []string
|
||||
Pattern *regexp.Regexp
|
||||
SecretGroup int
|
||||
Entropy *float64
|
||||
Allowlist Allowlist
|
||||
SecretType SecretType
|
||||
}
|
||||
|
||||
type Chunk struct {
|
||||
Content string
|
||||
FilePath string
|
||||
LineStart int
|
||||
CommitSHA string
|
||||
Author string
|
||||
CommitDate time.Time
|
||||
}
|
||||
|
||||
type Finding struct {
|
||||
RuleID string
|
||||
Description string
|
||||
Severity Severity
|
||||
Match string
|
||||
Secret string //nolint:gosec
|
||||
Entropy float64
|
||||
FilePath string
|
||||
LineNumber int
|
||||
LineContent string
|
||||
CommitSHA string
|
||||
Author string
|
||||
CommitDate time.Time
|
||||
HIBPStatus HIBPStatus
|
||||
BreachCount int
|
||||
}
|
||||
|
||||
type ScanResult struct {
|
||||
Findings []Finding
|
||||
TotalFiles int
|
||||
TotalRules int
|
||||
TotalFindings int
|
||||
HIBPChecked int
|
||||
HIBPBreached int
|
||||
Duration time.Duration
|
||||
}
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
aws_key = "AKIAIOSFODNN7EXAMPLE"
|
||||
password = "xK9mP2vL5nQ8jR3t"
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
database:
|
||||
url: "postgres://dbuser:s3cRetP4ssw0rd@rds.acmecorp.io:5432/appdata"
|
||||
|
|
@ -0,0 +1 @@
|
|||
const token = "ghp_R4nd0mT0k3nV4lu3H3r3Abcdef1234567890";
|
||||
|
|
@ -0,0 +1 @@
|
|||
-----BEGIN RSA PRIVATE KEY-----
|
||||
|
|
@ -0,0 +1 @@
|
|||
stripe_key = sk_live_4eC39HqLyjWDarjtT1zdp7dc
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
username = "admin"
|
||||
debug = true
|
||||
port = 8080
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
API_KEY=${VAULT_API_KEY}
|
||||
SECRET={{SECRET_FROM_VAULT}}
|
||||
TOKEN=REPLACE_ME
|
||||
|
|
@ -23,7 +23,7 @@
|
|||
<h2 align="center"><strong>View Complete Projects:</strong></h2>
|
||||
<div align="center">
|
||||
<a href="https://github.com/CarterPerez-dev/Cybersecurity-Projects/tree/main/PROJECTS">
|
||||
<img src="https://img.shields.io/badge/Full_Source_Code-15/60-blue?style=for-the-badge&logo=github" alt="Projects"/>
|
||||
<img src="https://img.shields.io/badge/Full_Source_Code-16/60-blue?style=for-the-badge&logo=github" alt="Projects"/>
|
||||
</a>
|
||||
</div>
|
||||
|
||||
|
|
@ -81,7 +81,7 @@ Tools, courses, certifications, communities, and frameworks for cybersecurity pr
|
|||
| **[OAuth Token Analyzer](./SYNOPSES/intermediate/OAuth.Token.Analyzer.md)**<br>Decode and validate JWT |    | JWT tokens • OAuth vulnerabilities • Signature validation<br>[Learn More](./SYNOPSES/intermediate/OAuth.Token.Analyzer.md) |
|
||||
| **[Web Vulnerability Scanner](./SYNOPSES/intermediate/Web.Vulnerability.Scanner.md)**<br>Automated XSS, SQLi, CSRF testing |    | Web vulnerabilities • Async scanning • Plugin architecture<br>[Learn More](./SYNOPSES/intermediate/Web.Vulnerability.Scanner.md) |
|
||||
| **[DDoS Mitigation Tool](./SYNOPSES/intermediate/DDoS.Mitigation.Tool.md)**<br>Detect traffic spikes |    | DDoS detection • Rate limiting • Anomaly detection<br>[Learn More](./SYNOPSES/intermediate/DDoS.Mitigation.Tool.md) |
|
||||
| **[Container Security Scanner](./SYNOPSES/intermediate/Container.Security.Scanner.md)**<br>Scan Docker misconfigurations |     | Container security • Dockerfile analysis • Docker API<br>[Learn More](./SYNOPSES/intermediate/Container.Security.Scanner.md) |
|
||||
| **[Secrets Scanner](./PROJECTS/intermediate/secrets-scanner)**<br>Scan codebases and git history for leaked secrets |    | Secret detection • Shannon entropy • HIBP k-anonymity • SARIF output<br>[Source Code](./PROJECTS/intermediate/secrets-scanner) \| [Docs](./PROJECTS/intermediate/secrets-scanner/learn) |
|
||||
| **[API Security Scanner](./PROJECTS/intermediate/api-security-scanner)**<br>Enterprise API vulnerability scanner |      | OWASP API Top 10 • ML fuzzing • GraphQL/SOAP testing<br>[Source Code](./PROJECTS/intermediate/api-security-scanner) \| [Docs](./PROJECTS/intermediate/api-security-scanner/learn) |
|
||||
| **[Wireless Deauth Detector](./SYNOPSES/intermediate/Wireless.Deauth.Detector.md)**<br>Monitor WiFi deauth attacks |    | Wireless security • Packet sniffing • Attack detection<br>[Learn More](./SYNOPSES/intermediate/Wireless.Deauth.Detector.md) |
|
||||
| **[Active Directory Enumeration](./SYNOPSES/intermediate/Active.Directory.Enumeration.md)**<br>Enumerate AD infrastructure |    | LDAP queries • AD structure • Privilege analysis<br>[Learn More](./SYNOPSES/intermediate/Active.Directory.Enumeration.md) |
|
||||
|
|
|
|||
|
|
@ -0,0 +1,140 @@
|
|||
# ©AngelaMos | 2026
|
||||
# .dockerignore.example
|
||||
#
|
||||
# Annotated .dockerignore template for Cybersecurity-Projects.
|
||||
# Copy into the same directory as your Dockerfile (usually project root
|
||||
# or backend/), rename to `.dockerignore`.
|
||||
#
|
||||
# Every file NOT listed here gets sent to the Docker daemon as build context.
|
||||
# A bloated context means slower builds and larger images. A missing
|
||||
# .dockerignore can accidentally bake secrets (.env) or gigabytes of
|
||||
# node_modules into your image.
|
||||
#
|
||||
# Syntax is identical to .gitignore (globs, negation with !).
|
||||
# Docs: https://docs.docker.com/build/concepts/context/#dockerignore-files
|
||||
|
||||
|
||||
# ── Virtual Environments ──────────────────────────────────────────────────────
|
||||
# These are host-specific. The container installs its own deps.
|
||||
|
||||
.venv/
|
||||
venv/
|
||||
|
||||
|
||||
# ── Python Caches ────────────────────────────────────────────────────────────
|
||||
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
*.pyo
|
||||
|
||||
|
||||
# ── Linter / Type Checker Caches ────────────────────────────────────────────
|
||||
|
||||
.mypy_cache/
|
||||
.ruff_cache/
|
||||
.pytest_cache/
|
||||
.ty_cache/
|
||||
|
||||
|
||||
# ── Test & Coverage Artifacts ────────────────────────────────────────────────
|
||||
# Tests run on the host or in a separate CI container, not in prod.
|
||||
|
||||
.coverage
|
||||
.coverage.*
|
||||
htmlcov/
|
||||
coverage.xml
|
||||
*.cover
|
||||
tests/
|
||||
conftest.py
|
||||
|
||||
|
||||
# ── Build Artifacts ──────────────────────────────────────────────────────────
|
||||
|
||||
dist/
|
||||
build/
|
||||
*.egg-info/
|
||||
*.egg
|
||||
|
||||
|
||||
# ── Node Modules ─────────────────────────────────────────────────────────────
|
||||
# Frontend builds in its own stage. Never copy host node_modules.
|
||||
|
||||
node_modules/
|
||||
.pnpm-store/
|
||||
|
||||
|
||||
# ── Secrets & Environment ────────────────────────────────────────────────────
|
||||
# NEVER bake secrets into an image. Pass them at runtime via env vars or
|
||||
# Docker secrets. The .env.example is fine to include for documentation.
|
||||
|
||||
.env
|
||||
.env.*
|
||||
!.env.example
|
||||
*.pem
|
||||
*.key
|
||||
|
||||
|
||||
# ── Git ──────────────────────────────────────────────────────────────────────
|
||||
# Git history has no business in a container image.
|
||||
|
||||
.git/
|
||||
.gitignore
|
||||
.gitattributes
|
||||
|
||||
|
||||
# ── Docker Files ─────────────────────────────────────────────────────────────
|
||||
# The Dockerfile itself and compose files aren't needed inside the image.
|
||||
|
||||
Dockerfile*
|
||||
docker-compose*
|
||||
compose*.yml
|
||||
dev.compose.yml
|
||||
.dockerignore
|
||||
|
||||
|
||||
# ── IDE / Editor ─────────────────────────────────────────────────────────────
|
||||
|
||||
.idea/
|
||||
.vscode/
|
||||
.zed/
|
||||
*.swp
|
||||
*.swo
|
||||
*~
|
||||
|
||||
|
||||
# ── Documentation ────────────────────────────────────────────────────────────
|
||||
# Docs are for humans, not containers.
|
||||
|
||||
docs/
|
||||
learn/
|
||||
*.md
|
||||
*.rst
|
||||
LICENSE
|
||||
CONTRIBUTING*
|
||||
|
||||
|
||||
# ── CI / Config ──────────────────────────────────────────────────────────────
|
||||
|
||||
.github/
|
||||
.pre-commit-config.yaml
|
||||
.editorconfig
|
||||
justfile
|
||||
Justfile
|
||||
Makefile
|
||||
|
||||
|
||||
# ── Logs & Databases ────────────────────────────────────────────────────────
|
||||
|
||||
*.log
|
||||
logs/
|
||||
*.db
|
||||
*.sqlite
|
||||
*.sqlite3
|
||||
|
||||
|
||||
# ── OS Junk ──────────────────────────────────────────────────────────────────
|
||||
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
*.bak
|
||||
|
|
@ -0,0 +1,119 @@
|
|||
# ©AngelaMos | 2026
|
||||
# .editorconfig.example
|
||||
#
|
||||
# Annotated EditorConfig template for Cybersecurity-Projects.
|
||||
# Copy into your project ROOT, rename to `.editorconfig`.
|
||||
# No plugins needed — VS Code, JetBrains, Neovim, and Zed support it natively.
|
||||
#
|
||||
# EditorConfig enforces consistent whitespace and encoding across every editor
|
||||
# and developer machine. It is NOT a replacement for formatters (ruff, biome,
|
||||
# gofmt) — it's the baseline that ensures files are sane before formatters run.
|
||||
#
|
||||
# Docs: https://editorconfig.org
|
||||
# Spec: https://spec.editorconfig.org
|
||||
|
||||
|
||||
# ── Root Marker ───────────────────────────────────────────────────────────────
|
||||
# `root = true` stops EditorConfig from searching parent directories.
|
||||
# Always set this in the project root so configs don't leak between repos.
|
||||
|
||||
root = true
|
||||
|
||||
|
||||
# ── Global Defaults ───────────────────────────────────────────────────────────
|
||||
# These apply to EVERY file unless overridden by a more specific section below.
|
||||
#
|
||||
# end_of_line = lf
|
||||
# Use LF everywhere (even on Windows). Git handles CRLF conversion via
|
||||
# .gitattributes — the working tree should always be LF.
|
||||
#
|
||||
# insert_final_newline = true
|
||||
# POSIX standard. Prevents "No newline at end of file" diffs.
|
||||
#
|
||||
# trim_trailing_whitespace = true
|
||||
# Removes invisible trailing spaces that cause noisy diffs.
|
||||
|
||||
[*]
|
||||
indent_style = space
|
||||
indent_size = 2
|
||||
end_of_line = lf
|
||||
charset = utf-8
|
||||
trim_trailing_whitespace = true
|
||||
insert_final_newline = true
|
||||
|
||||
|
||||
# ── Python ────────────────────────────────────────────────────────────────────
|
||||
# PEP 8: 4-space indentation.
|
||||
|
||||
[*.py]
|
||||
indent_size = 4
|
||||
|
||||
|
||||
# ── Go ────────────────────────────────────────────────────────────────────────
|
||||
# gofmt enforces tabs. This is non-negotiable in Go.
|
||||
|
||||
[*.go]
|
||||
indent_style = tab
|
||||
indent_size = 4
|
||||
|
||||
|
||||
# ── Makefile ──────────────────────────────────────────────────────────────────
|
||||
# Make REQUIRES hard tabs. Spaces will silently break your recipes.
|
||||
|
||||
[Makefile]
|
||||
indent_style = tab
|
||||
|
||||
[*.mk]
|
||||
indent_style = tab
|
||||
|
||||
|
||||
# ── YAML ──────────────────────────────────────────────────────────────────────
|
||||
# YAML is always 2-space. Tabs are illegal in YAML.
|
||||
|
||||
[*.{yml,yaml}]
|
||||
indent_size = 2
|
||||
|
||||
|
||||
# ── JavaScript / TypeScript ──────────────────────────────────────────────────
|
||||
# 2-space is the ecosystem standard (Node, React, Vite, Biome defaults).
|
||||
|
||||
[*.{js,jsx,ts,tsx,mjs,cjs}]
|
||||
indent_size = 2
|
||||
|
||||
|
||||
# ── JSON / TOML / Config Files ──────────────────────────────────────────────
|
||||
|
||||
[*.{json,jsonc,toml}]
|
||||
indent_size = 2
|
||||
|
||||
|
||||
# ── Markdown ──────────────────────────────────────────────────────────────────
|
||||
# Markdown uses trailing spaces for line breaks (<br>).
|
||||
# Do NOT trim them or you'll break intentional formatting.
|
||||
|
||||
[*.md]
|
||||
trim_trailing_whitespace = false
|
||||
|
||||
|
||||
# ── Dockerfiles ──────────────────────────────────────────────────────────────
|
||||
|
||||
[Dockerfile*]
|
||||
indent_size = 4
|
||||
|
||||
|
||||
# ── Shell Scripts ────────────────────────────────────────────────────────────
|
||||
|
||||
[*.{sh,bash,zsh}]
|
||||
indent_size = 4
|
||||
|
||||
|
||||
# ── C / C++ / Rust ──────────────────────────────────────────────────────────
|
||||
|
||||
[*.{c,h,cpp,hpp,rs}]
|
||||
indent_size = 4
|
||||
|
||||
|
||||
# ── Haskell ──────────────────────────────────────────────────────────────────
|
||||
|
||||
[*.hs]
|
||||
indent_size = 2
|
||||
|
|
@ -0,0 +1,150 @@
|
|||
# ©AngelaMos | 2026
|
||||
# .gitignore.example
|
||||
#
|
||||
# Annotated .gitignore template for Cybersecurity-Projects.
|
||||
# Copy into your project root and rename to .gitignore.
|
||||
# Delete any sections that don't apply to your project's stack.
|
||||
#
|
||||
# Pattern syntax:
|
||||
# file.txt exact filename anywhere in the repo
|
||||
# *.log glob — all .log files
|
||||
# /build leading slash — only match at repo root
|
||||
# build/ trailing slash — only match directories
|
||||
# !important negate — re-include something a prior rule excluded
|
||||
# **/cache match in any subdirectory
|
||||
#
|
||||
# Docs: https://git-scm.com/docs/gitignore
|
||||
# Generator: https://www.toptal.com/developers/gitignore
|
||||
|
||||
|
||||
# ── Secrets & Environment ─────────────────────────────────────────────────────
|
||||
# MOST IMPORTANT SECTION. Leaked secrets are the #1 gitignore failure mode.
|
||||
# If you accidentally commit a secret, rotating the credential is the only
|
||||
# safe fix — removing it from history alone is not enough.
|
||||
|
||||
.env
|
||||
.env.*
|
||||
!.env.example
|
||||
*.pem
|
||||
*.key
|
||||
*.p12
|
||||
*.pfx
|
||||
*.crt
|
||||
|
||||
|
||||
# ── Python ────────────────────────────────────────────────────────────────────
|
||||
|
||||
# Bytecode / compiled
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
*.so
|
||||
|
||||
# Virtual environments
|
||||
.venv/
|
||||
venv/
|
||||
|
||||
# Build & distribution (uv build / hatch build output)
|
||||
build/
|
||||
dist/
|
||||
*.egg-info/
|
||||
*.egg
|
||||
|
||||
# Type checker / linter caches
|
||||
.mypy_cache/
|
||||
.ruff_cache/
|
||||
.pytype/
|
||||
|
||||
# Test & coverage artifacts
|
||||
.pytest_cache/
|
||||
.coverage
|
||||
.coverage.*
|
||||
htmlcov/
|
||||
*.prof
|
||||
|
||||
|
||||
# ── Go ────────────────────────────────────────────────────────────────────────
|
||||
|
||||
# Compiled binary (Go produces a single binary, name varies per project)
|
||||
# /bin/
|
||||
# /<your-binary-name>
|
||||
|
||||
# Debug binary from dlv
|
||||
__debug_bin*
|
||||
|
||||
# Profiling
|
||||
*.prof
|
||||
*.test
|
||||
trace.out
|
||||
coverage.out
|
||||
|
||||
|
||||
# ── Node / JavaScript ────────────────────────────────────────────────────────
|
||||
|
||||
node_modules/
|
||||
|
||||
# pnpm
|
||||
.pnpm-store/
|
||||
|
||||
# Build output (Vite, Next.js, etc.)
|
||||
.next/
|
||||
.nuxt/
|
||||
.output/
|
||||
|
||||
|
||||
# ── Docker ────────────────────────────────────────────────────────────────────
|
||||
|
||||
# Override files are machine-specific and should not be shared
|
||||
docker-compose.override.yml
|
||||
docker-compose.override.yaml
|
||||
|
||||
|
||||
# ── Databases (local dev only) ────────────────────────────────────────────────
|
||||
|
||||
*.db
|
||||
*.sqlite
|
||||
*.sqlite3
|
||||
dump.rdb
|
||||
|
||||
|
||||
# ── IDE / Editor ──────────────────────────────────────────────────────────────
|
||||
|
||||
# VS Code (settings.json and extensions.json are sometimes committed — your call)
|
||||
.vscode/
|
||||
|
||||
# JetBrains (IntelliJ, GoLand, PyCharm, WebStorm)
|
||||
.idea/
|
||||
|
||||
# Vim / Neovim swap & backup
|
||||
*.swp
|
||||
*.swo
|
||||
*~
|
||||
|
||||
# Zed
|
||||
.zed/
|
||||
|
||||
|
||||
# ── OS Junk ───────────────────────────────────────────────────────────────────
|
||||
|
||||
# macOS
|
||||
.DS_Store
|
||||
|
||||
# Windows
|
||||
Thumbs.db
|
||||
desktop.ini
|
||||
|
||||
# Linux (some file managers)
|
||||
.Trash-*
|
||||
|
||||
|
||||
# ── Logs ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
*.log
|
||||
logs/
|
||||
|
||||
|
||||
# ── Binaries & Compiled Output ────────────────────────────────────────────────
|
||||
|
||||
*.exe
|
||||
*.dll
|
||||
*.dylib
|
||||
|
|
@ -0,0 +1,113 @@
|
|||
# AngelaMos | 2026
|
||||
# .golangci.yml
|
||||
|
||||
version: "2"
|
||||
|
||||
linters:
|
||||
default: none
|
||||
enable:
|
||||
- errcheck
|
||||
- govet
|
||||
- gosec
|
||||
- bodyclose
|
||||
- nilerr
|
||||
- errorlint
|
||||
- exhaustive
|
||||
- gocritic
|
||||
- funlen
|
||||
- gocognit
|
||||
- dupl
|
||||
- goconst
|
||||
- ineffassign
|
||||
- unused
|
||||
- unconvert
|
||||
- unparam
|
||||
- testifylint
|
||||
- fatcontext
|
||||
|
||||
settings:
|
||||
errcheck:
|
||||
check-type-assertions: true
|
||||
check-blank: true
|
||||
|
||||
funlen:
|
||||
lines: 100
|
||||
statements: 50
|
||||
|
||||
gocognit:
|
||||
min-complexity: 20
|
||||
|
||||
govet:
|
||||
enable-all: true
|
||||
disable:
|
||||
- fieldalignment
|
||||
|
||||
revive:
|
||||
rules:
|
||||
- name: blank-imports
|
||||
- name: context-as-argument
|
||||
- name: context-keys-type
|
||||
- name: error-return
|
||||
- name: error-strings
|
||||
- name: error-naming
|
||||
- name: exported
|
||||
- name: increment-decrement
|
||||
- name: var-declaration
|
||||
- name: package-comments
|
||||
disabled: true
|
||||
- name: range
|
||||
- name: receiver-naming
|
||||
- name: time-naming
|
||||
- name: unexported-return
|
||||
- name: indent-error-flow
|
||||
- name: errorf
|
||||
- name: empty-block
|
||||
- name: superfluous-else
|
||||
- name: unreachable-code
|
||||
|
||||
staticcheck:
|
||||
checks:
|
||||
- all
|
||||
|
||||
gosec:
|
||||
excludes:
|
||||
- G104
|
||||
|
||||
sloglint:
|
||||
no-mixed-args: true
|
||||
kv-only: true
|
||||
context: all
|
||||
|
||||
issues:
|
||||
max-same-issues: 50
|
||||
exclude-dirs:
|
||||
- vendor
|
||||
- testdata
|
||||
exclude-rules:
|
||||
- path: _test\.go
|
||||
linters:
|
||||
- funlen
|
||||
- dupl
|
||||
- goconst
|
||||
|
||||
|
||||
formatters:
|
||||
enable:
|
||||
- gci # Groups imports
|
||||
- gofumpt # Whitespace
|
||||
- golines # Vertical wrap
|
||||
settings:
|
||||
golines:
|
||||
max-len: 80
|
||||
reformat-tags: true
|
||||
goimports:
|
||||
local-prefixes:
|
||||
- github.com/GITHUB-USERNAME-HERE/LOCAL-IMPORT-NAME-HERE
|
||||
gci:
|
||||
sections:
|
||||
- standard
|
||||
- default
|
||||
- prefix(github.com/GITHUB-USERNAME-HERE)
|
||||
custom-order: true
|
||||
gofumpt:
|
||||
extra-rules: true
|
||||
|
|
@ -0,0 +1,151 @@
|
|||
# ©AngelaMos | 2026
|
||||
# .pre-commit-config.yaml.example
|
||||
#
|
||||
# Annotated pre-commit configuration for Cybersecurity-Projects.
|
||||
# Copy into your project root, rename to `.pre-commit-config.yaml`.
|
||||
# Delete hooks that don't apply to your project's stack.
|
||||
#
|
||||
# pre-commit runs checks automatically on `git commit`. If any hook fails,
|
||||
# the commit is blocked until you fix the issue — catching problems before
|
||||
# they hit CI saves everyone time.
|
||||
#
|
||||
# Install:
|
||||
# uv tool install pre-commit (recommended — global, no venv pollution)
|
||||
# pipx install pre-commit (alternative)
|
||||
#
|
||||
# Setup (run once per clone):
|
||||
# pre-commit install hooks into .git/hooks/pre-commit
|
||||
# pre-commit install --hook-type commit-msg (optional: commit msg hooks)
|
||||
#
|
||||
# Usage:
|
||||
# git commit hooks run automatically on staged files
|
||||
# pre-commit run --all-files run all hooks on every file (CI style)
|
||||
# pre-commit autoupdate bump hook versions to latest tags
|
||||
#
|
||||
# Docs: https://pre-commit.com
|
||||
# Hook search: https://pre-commit.com/hooks.html
|
||||
|
||||
|
||||
# ── General File Hygiene ──────────────────────────────────────────────────────
|
||||
# These lightweight hooks catch common mistakes. They run in milliseconds.
|
||||
|
||||
repos:
|
||||
- repo: https://github.com/pre-commit/pre-commit-hooks
|
||||
rev: v5.0.0
|
||||
hooks:
|
||||
# Fails if a file has syntax-invalid TOML (catches pyproject.toml typos).
|
||||
- id: check-toml
|
||||
|
||||
# Fails if any YAML file has syntax errors.
|
||||
- id: check-yaml
|
||||
args: [--allow-multiple-documents]
|
||||
|
||||
# Fails if any JSON file has syntax errors.
|
||||
- id: check-json
|
||||
|
||||
# Catches leftover `breakpoint()`, `pdb.set_trace()`, `debugger` statements.
|
||||
- id: debug-statements
|
||||
|
||||
# Ensures every file ends with exactly one newline (POSIX standard).
|
||||
- id: end-of-file-fixer
|
||||
|
||||
# Strips trailing whitespace from lines.
|
||||
- id: trailing-whitespace
|
||||
args: [--markdown-linebreak-ext=md]
|
||||
|
||||
# Prevents accidentally committing large files (default 500KB).
|
||||
- id: check-added-large-files
|
||||
args: [--maxkb=500]
|
||||
|
||||
# Prevents accidentally committing private keys.
|
||||
- id: detect-private-key
|
||||
|
||||
# Fails if files would conflict on case-insensitive filesystems (macOS/Windows).
|
||||
- id: check-case-conflict
|
||||
|
||||
# Prevents committing directly to main/master.
|
||||
- id: no-commit-to-branch
|
||||
args: [--branch, main, --branch, master]
|
||||
|
||||
|
||||
# ── Python: Ruff (linter) ──────────────────────────────────────────────────
|
||||
# Runs ruff check on staged Python files. Fast (Rust-based), replaces
|
||||
# flake8 + isort + pyupgrade + dozens of plugins in a single tool.
|
||||
#
|
||||
# Change the `args` path to match your project layout:
|
||||
# Python-only: args: ["src/", "tests/"]
|
||||
# Fullstack: args: ["backend/"]
|
||||
|
||||
- repo: https://github.com/astral-sh/ruff-pre-commit
|
||||
rev: v0.15.1
|
||||
hooks:
|
||||
- id: ruff
|
||||
args: ["<src>/"]
|
||||
always_run: true
|
||||
|
||||
|
||||
# ── Python: Local Hooks ─────────────────────────────────────────────────────
|
||||
# `repo: local` hooks run tools installed in your project's venv.
|
||||
# This means they use the EXACT versions pinned in pyproject.toml.
|
||||
#
|
||||
# `pass_filenames: false` + `always_run: true` = runs on the full project
|
||||
# every time, not just on staged files. Slower but catches cross-file issues.
|
||||
#
|
||||
# Adjust `entry` paths to match your layout.
|
||||
|
||||
- repo: local
|
||||
hooks:
|
||||
- id: mypy-check
|
||||
name: mypy type checking
|
||||
entry: bash -c 'cd <src-root> && mypy .'
|
||||
language: system
|
||||
types: [python]
|
||||
pass_filenames: false
|
||||
always_run: true
|
||||
|
||||
- id: pylint-check
|
||||
name: pylint code quality
|
||||
entry: bash -c 'cd <src-root> && pylint .'
|
||||
language: system
|
||||
types: [python]
|
||||
pass_filenames: false
|
||||
always_run: true
|
||||
|
||||
|
||||
# ── Frontend: Biome (uncomment for fullstack projects) ──────────────────────
|
||||
# Biome handles JS/TS linting + formatting in one pass.
|
||||
# Requires Biome installed in frontend/node_modules.
|
||||
#
|
||||
# - repo: local
|
||||
# hooks:
|
||||
# - id: biome-check
|
||||
# name: biome lint + format check
|
||||
# entry: bash -c 'cd frontend && pnpm biome check .'
|
||||
# language: system
|
||||
# types_or: [javascript, jsx, ts, tsx, json]
|
||||
# pass_filenames: false
|
||||
# always_run: true
|
||||
#
|
||||
# - id: tsc-check
|
||||
# name: typescript type check
|
||||
# entry: bash -c 'cd frontend && pnpm tsc --noEmit'
|
||||
# language: system
|
||||
# types_or: [ts, tsx]
|
||||
# pass_filenames: false
|
||||
# always_run: true
|
||||
|
||||
|
||||
# ── Go (uncomment for Go projects) ─────────────────────────────────────────
|
||||
# - repo: https://github.com/golangci/golangci-lint
|
||||
# rev: v2.1.0
|
||||
# hooks:
|
||||
# - id: golangci-lint
|
||||
#
|
||||
# - repo: local
|
||||
# hooks:
|
||||
# - id: go-vet
|
||||
# name: go vet
|
||||
# entry: go vet ./...
|
||||
# language: system
|
||||
# types: [go]
|
||||
# pass_filenames: false
|
||||
|
|
@ -0,0 +1,251 @@
|
|||
# ©AngelaMos | 2026
|
||||
# Makefile.example
|
||||
#
|
||||
# Annotated Makefile template for Cybersecurity-Projects.
|
||||
# Copy into your project root, rename to `Makefile`, and replace
|
||||
# every <PLACEHOLDER> with your actual values. Delete sections
|
||||
# that don't apply to your project's stack.
|
||||
#
|
||||
# Make is pre-installed on virtually every Unix system — no extra tooling.
|
||||
# Use this when you want zero dependencies for task running.
|
||||
#
|
||||
# Docs: https://www.gnu.org/software/make/manual/make.html
|
||||
#
|
||||
# Usage:
|
||||
# make show available targets (self-documenting help)
|
||||
# make <target> run a target
|
||||
# make -n <target> dry-run (print commands without executing)
|
||||
# make -j4 <target> run up to 4 jobs in parallel
|
||||
#
|
||||
# IMPORTANT: Makefiles require TABS for indentation, not spaces.
|
||||
# If your editor converts tabs to spaces, this file will break.
|
||||
|
||||
|
||||
# ── Variables ─────────────────────────────────────────────────────────────────
|
||||
# Change these to match your project layout.
|
||||
# ?= means "set only if not already defined" — overridable from the CLI:
|
||||
# make lint SRC_DIR=app/
|
||||
|
||||
PROJECT_NAME ?= <project-name>
|
||||
SRC_DIR ?= <src>
|
||||
TEST_DIR ?= <tests>
|
||||
PACKAGE ?= <package-name>
|
||||
PYTHON ?= python3
|
||||
DOCKER_COMP ?= docker compose
|
||||
DEV_COMPOSE ?= docker compose -f dev.compose.yml
|
||||
|
||||
|
||||
# ── Self-Documenting Help ─────────────────────────────────────────────────────
|
||||
# Any line with `## comment` after a target becomes part of `make help`.
|
||||
# This is the standard pattern for self-documenting Makefiles.
|
||||
# `make` with no args runs the first target, which is `help`.
|
||||
|
||||
.DEFAULT_GOAL := help
|
||||
|
||||
.PHONY: help
|
||||
help: ## Show this help message
|
||||
@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | \
|
||||
awk 'BEGIN {FS = ":.*?## "}; {printf " \033[36m%-20s\033[0m %s\n", $$1, $$2}'
|
||||
|
||||
|
||||
# ── Linting and Formatting ───────────────────────────────────────────────────
|
||||
|
||||
.PHONY: ruff ruff-fix ruff-format pylint lint
|
||||
|
||||
ruff: ## Run ruff linter
|
||||
ruff check $(SRC_DIR)/ $(TEST_DIR)/
|
||||
|
||||
ruff-fix: ## Run ruff with auto-fix + format
|
||||
ruff check $(SRC_DIR)/ $(TEST_DIR)/ --fix
|
||||
ruff format $(SRC_DIR)/ $(TEST_DIR)/
|
||||
|
||||
ruff-format: ## Format code with ruff
|
||||
ruff format $(SRC_DIR)/ $(TEST_DIR)/
|
||||
|
||||
pylint: ## Run pylint
|
||||
pylint $(SRC_DIR)/$(PACKAGE)
|
||||
|
||||
lint: ruff pylint ## Run all Python linters
|
||||
|
||||
|
||||
# ── Frontend Linting (delete for Python-only projects) ──────────────────────
|
||||
|
||||
.PHONY: biome biome-fix stylelint stylelint-fix tsc
|
||||
|
||||
biome: ## Run Biome linter on frontend
|
||||
cd frontend && pnpm biome check .
|
||||
|
||||
biome-fix: ## Run Biome with auto-fix
|
||||
cd frontend && pnpm biome check --write .
|
||||
|
||||
stylelint: ## Lint SCSS files
|
||||
cd frontend && pnpm stylelint '**/*.scss'
|
||||
|
||||
stylelint-fix: ## Fix SCSS lint issues
|
||||
cd frontend && pnpm stylelint '**/*.scss' --fix
|
||||
|
||||
tsc: ## TypeScript type check (no emit)
|
||||
cd frontend && pnpm tsc --noEmit
|
||||
|
||||
|
||||
# ── Type Checking ────────────────────────────────────────────────────────────
|
||||
|
||||
.PHONY: mypy typecheck
|
||||
|
||||
mypy: ## Run mypy type checker
|
||||
mypy $(SRC_DIR)/
|
||||
|
||||
typecheck: mypy ## Alias for mypy
|
||||
|
||||
|
||||
# ── Testing ──────────────────────────────────────────────────────────────────
|
||||
|
||||
.PHONY: test test-cov
|
||||
|
||||
test: ## Run tests
|
||||
pytest $(TEST_DIR)/
|
||||
|
||||
test-cov: ## Run tests with coverage report
|
||||
pytest $(TEST_DIR)/ --cov=$(SRC_DIR)/$(PACKAGE) --cov-report=term-missing --cov-report=html
|
||||
|
||||
|
||||
# ── CI / Quality ─────────────────────────────────────────────────────────────
|
||||
# Composite targets that chain multiple checks.
|
||||
|
||||
.PHONY: ci check
|
||||
|
||||
ci: lint typecheck test ## Full CI pipeline (lint + types + tests)
|
||||
|
||||
check: ruff mypy ## Fast check (lint + types, no tests)
|
||||
|
||||
|
||||
# ── Docker Compose — Production (delete if no Docker) ───────────────────────
|
||||
|
||||
.PHONY: up start down stop build rebuild logs ps shell
|
||||
|
||||
up: ## Start services (foreground)
|
||||
$(DOCKER_COMP) up
|
||||
|
||||
start: ## Start services (detached)
|
||||
$(DOCKER_COMP) up -d
|
||||
|
||||
down: ## Stop and remove containers
|
||||
$(DOCKER_COMP) down
|
||||
|
||||
stop: ## Stop containers (keep state)
|
||||
$(DOCKER_COMP) stop
|
||||
|
||||
build: ## Build container images
|
||||
$(DOCKER_COMP) build
|
||||
|
||||
rebuild: ## Rebuild without cache
|
||||
$(DOCKER_COMP) build --no-cache
|
||||
|
||||
logs: ## Tail service logs
|
||||
$(DOCKER_COMP) logs -f
|
||||
|
||||
ps: ## List running containers
|
||||
$(DOCKER_COMP) ps
|
||||
|
||||
shell: ## Shell into backend container
|
||||
$(DOCKER_COMP) exec -it backend /bin/bash
|
||||
|
||||
|
||||
# ── Docker Compose — Dev (delete if no Docker) ─────────────────────────────
|
||||
|
||||
.PHONY: dev-up dev-start dev-down dev-stop dev-build dev-rebuild dev-logs dev-ps dev-shell
|
||||
|
||||
dev-up: ## Start dev services (foreground)
|
||||
$(DEV_COMPOSE) up
|
||||
|
||||
dev-start: ## Start dev services (detached)
|
||||
$(DEV_COMPOSE) up -d
|
||||
|
||||
dev-down: ## Stop dev containers
|
||||
$(DEV_COMPOSE) down
|
||||
|
||||
dev-stop: ## Stop dev containers (keep state)
|
||||
$(DEV_COMPOSE) stop
|
||||
|
||||
dev-build: ## Build dev images
|
||||
$(DEV_COMPOSE) build
|
||||
|
||||
dev-rebuild: ## Rebuild dev images without cache
|
||||
$(DEV_COMPOSE) build --no-cache
|
||||
|
||||
dev-logs: ## Tail dev logs
|
||||
$(DEV_COMPOSE) logs -f
|
||||
|
||||
dev-ps: ## List dev containers
|
||||
$(DEV_COMPOSE) ps
|
||||
|
||||
dev-shell: ## Shell into dev backend
|
||||
$(DEV_COMPOSE) exec -it backend /bin/bash
|
||||
|
||||
|
||||
# ── Database (delete if no database) ────────────────────────────────────────
|
||||
|
||||
.PHONY: migrate migration rollback db-history db-current
|
||||
|
||||
migrate: ## Run database migrations (Docker)
|
||||
$(DOCKER_COMP) exec backend alembic upgrade head
|
||||
|
||||
migration: ## Create new migration (usage: make migration MSG="add users table")
|
||||
$(DOCKER_COMP) exec backend alembic revision --autogenerate -m "$(MSG)"
|
||||
|
||||
rollback: ## Rollback last migration
|
||||
$(DOCKER_COMP) exec backend alembic downgrade -1
|
||||
|
||||
db-history: ## Show migration history
|
||||
$(DOCKER_COMP) exec backend alembic history --verbose
|
||||
|
||||
db-current: ## Show current migration
|
||||
$(DOCKER_COMP) exec backend alembic current
|
||||
|
||||
|
||||
# ── Setup ────────────────────────────────────────────────────────────────────
|
||||
|
||||
.PHONY: setup setup-backend setup-frontend env
|
||||
|
||||
setup: ## First-time project setup
|
||||
@echo "Setting up development environment..."
|
||||
uv sync --all-extras
|
||||
@echo "Setup complete!"
|
||||
|
||||
# Fullstack alternative (uncomment and replace the simple setup above):
|
||||
# setup: setup-backend setup-frontend env ## First-time project setup
|
||||
# @echo "Setup complete!"
|
||||
#
|
||||
# setup-backend:
|
||||
# @echo "Setting up backend..."
|
||||
# cd backend && uv sync
|
||||
#
|
||||
# setup-frontend:
|
||||
# @echo "Setting up frontend..."
|
||||
# cd frontend && pnpm install
|
||||
#
|
||||
# env:
|
||||
# @test -f .env || (cp .env.example .env && echo "Created .env from .env.example")
|
||||
# @echo ".env ready — update with your values."
|
||||
|
||||
|
||||
# ── Utilities ────────────────────────────────────────────────────────────────
|
||||
|
||||
.PHONY: info clean nuke
|
||||
|
||||
info: ## Show project info
|
||||
@echo "Project: $(PROJECT_NAME)"
|
||||
@echo "Python: $(shell $(PYTHON) --version 2>/dev/null || echo 'not found')"
|
||||
@echo "OS: $(shell uname -s) ($(shell uname -m))"
|
||||
|
||||
clean: ## Remove caches and build artifacts
|
||||
rm -rf .mypy_cache .pytest_cache .ruff_cache htmlcov dist build
|
||||
rm -f .coverage
|
||||
@echo "Cache directories cleaned"
|
||||
|
||||
nuke: ## Remove everything including venv (prompts for confirmation)
|
||||
@echo "This will remove .venv, all caches, and build artifacts."
|
||||
@read -p "Continue? [y/N] " confirm && [ "$$confirm" = "y" ] || exit 1
|
||||
rm -rf .mypy_cache .pytest_cache .ruff_cache htmlcov dist build .venv
|
||||
rm -f .coverage
|
||||
@echo "Nuke complete!"
|
||||
|
|
@ -0,0 +1,157 @@
|
|||
{
|
||||
"//": [
|
||||
"©AngelaMos | 2026",
|
||||
"biome.json.example",
|
||||
"",
|
||||
"Annotated Biome configuration for Cybersecurity-Projects frontends.",
|
||||
"Copy into your frontend/ directory, rename to biome.json.",
|
||||
"",
|
||||
"Biome is an all-in-one linter + formatter for JS/TS/JSON/CSS.",
|
||||
"It replaces ESLint + Prettier in a single Rust-based binary.",
|
||||
"",
|
||||
"Docs: https://biomejs.dev/reference/configuration/",
|
||||
"Rules: https://biomejs.dev/linter/rules/"
|
||||
],
|
||||
|
||||
"$schema": "https://biomejs.dev/schemas/2.4.2/schema.json",
|
||||
|
||||
"// vcs": "Integrates with Git so Biome respects your .gitignore automatically.",
|
||||
"vcs": {
|
||||
"enabled": true,
|
||||
"clientKind": "git",
|
||||
"useIgnoreFile": true
|
||||
},
|
||||
|
||||
"// files": "Only process source files. Biome auto-skips lock files and node_modules.",
|
||||
"files": {
|
||||
"includes": ["**/*.ts", "**/*.tsx", "**/*.js", "**/*.jsx", "**/*.json"]
|
||||
},
|
||||
|
||||
"// formatter": [
|
||||
"Global formatting rules. These apply to all languages.",
|
||||
"indentWidth 2 and lineWidth 82 match the repo standard.",
|
||||
"lineEnding 'lf' enforces Unix-style line endings everywhere."
|
||||
],
|
||||
"formatter": {
|
||||
"enabled": true,
|
||||
"indentStyle": "space",
|
||||
"indentWidth": 2,
|
||||
"lineWidth": 82,
|
||||
"lineEnding": "lf"
|
||||
},
|
||||
|
||||
"// javascript": [
|
||||
"JS/TS-specific formatter settings.",
|
||||
"quoteStyle 'single' for JS, 'double' for JSX (React convention).",
|
||||
"semicolons 'asNeeded' = no semis except where ASI would be ambiguous.",
|
||||
"trailingCommas 'es5' = trailing commas in arrays/objects but not function params."
|
||||
],
|
||||
"javascript": {
|
||||
"formatter": {
|
||||
"quoteStyle": "single",
|
||||
"jsxQuoteStyle": "double",
|
||||
"semicolons": "asNeeded",
|
||||
"trailingCommas": "es5",
|
||||
"arrowParentheses": "always"
|
||||
}
|
||||
},
|
||||
|
||||
"// linter": [
|
||||
"Start with 'recommended' (curated safe defaults), then tune per-domain.",
|
||||
"Do NOT use 'all' — it enables 300+ conflicting rules."
|
||||
],
|
||||
"linter": {
|
||||
"enabled": true,
|
||||
"rules": {
|
||||
"recommended": true,
|
||||
|
||||
"// complexity": "Keeps code readable. noForEach off because .forEach is fine for side effects.",
|
||||
"complexity": {
|
||||
"noExcessiveCognitiveComplexity": {
|
||||
"level": "error",
|
||||
"options": { "maxAllowedComplexity": 25 }
|
||||
},
|
||||
"noForEach": "off",
|
||||
"useLiteralKeys": "off"
|
||||
},
|
||||
|
||||
"// correctness": "Catches real bugs. These should almost always be errors.",
|
||||
"correctness": {
|
||||
"noUnusedVariables": "error",
|
||||
"noUnusedImports": "error",
|
||||
"useExhaustiveDependencies": "warn",
|
||||
"useHookAtTopLevel": "error",
|
||||
"noUndeclaredVariables": "error"
|
||||
},
|
||||
|
||||
"// style": [
|
||||
"Enforces consistent patterns across the codebase.",
|
||||
"useImportType separates type imports (tree-shaking friendly).",
|
||||
"noNonNullAssertion bans `!` postfix (override in main.tsx via overrides).",
|
||||
"useNamingConvention is off — too noisy with API response shapes."
|
||||
],
|
||||
"style": {
|
||||
"useImportType": "error",
|
||||
"useConst": "error",
|
||||
"useTemplate": "error",
|
||||
"useSelfClosingElements": "error",
|
||||
"useFragmentSyntax": "error",
|
||||
"noNonNullAssertion": "error",
|
||||
"useConsistentArrayType": {
|
||||
"level": "error",
|
||||
"options": { "syntax": "shorthand" }
|
||||
},
|
||||
"useNamingConvention": "off"
|
||||
},
|
||||
|
||||
"// suspicious": [
|
||||
"Catches code that is almost certainly a mistake.",
|
||||
"noExplicitAny = error forces proper typing (the whole point of TS).",
|
||||
"noConsole = warn because console.log is fine in dev but should be flagged."
|
||||
],
|
||||
"suspicious": {
|
||||
"noExplicitAny": "error",
|
||||
"noDebugger": "error",
|
||||
"noConsole": "warn",
|
||||
"noArrayIndexKey": "warn",
|
||||
"noAssignInExpressions": "error",
|
||||
"noDoubleEquals": "error",
|
||||
"noRedeclare": "error",
|
||||
"noVar": "error"
|
||||
},
|
||||
|
||||
"// security": "Prevents XSS vectors in React components.",
|
||||
"security": {
|
||||
"noDangerouslySetInnerHtml": "error"
|
||||
},
|
||||
|
||||
"// a11y": "Accessibility rules. Non-negotiable for production UIs.",
|
||||
"a11y": {
|
||||
"useAltText": "error",
|
||||
"useAnchorContent": "error",
|
||||
"useKeyWithClickEvents": "error",
|
||||
"noStaticElementInteractions": "error",
|
||||
"useButtonType": "error",
|
||||
"useValidAnchor": "error"
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
"// overrides": [
|
||||
"Per-file rule overrides.",
|
||||
"main.tsx needs `document.getElementById('root')!` — the non-null assertion",
|
||||
"is safe here because the HTML template guarantees the element exists."
|
||||
],
|
||||
"overrides": [
|
||||
{
|
||||
"includes": ["src/main.tsx"],
|
||||
"linter": {
|
||||
"rules": {
|
||||
"style": {
|
||||
"noNonNullAssertion": "off"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,115 @@
|
|||
# ©AngelaMos | 2026
|
||||
# dependabot.yml.example
|
||||
#
|
||||
# Annotated Dependabot configuration for Cybersecurity-Projects.
|
||||
# Copy to `.github/dependabot.yml` in your repository.
|
||||
#
|
||||
# Dependabot automatically opens PRs to update outdated dependencies.
|
||||
# It's GitHub-native, free, and requires zero infrastructure.
|
||||
#
|
||||
# For a security-focused repo, keeping deps patched is non-negotiable.
|
||||
# Most CVEs are in transitive dependencies you never directly imported.
|
||||
#
|
||||
# Docs: https://docs.github.com/en/code-security/dependabot/dependabot-version-updates
|
||||
# Config: https://docs.github.com/en/code-security/dependabot/working-with-dependabot/dependabot-options-reference
|
||||
|
||||
version: 2
|
||||
|
||||
updates:
|
||||
|
||||
# ── Python (pip ecosystem) ────────────────────────────────────────────────
|
||||
# Covers pyproject.toml, requirements.txt, setup.py.
|
||||
# Use "pip" even if you use uv — Dependabot reads the manifest format,
|
||||
# not the package manager binary.
|
||||
#
|
||||
# directory: path to the manifest file relative to repo root.
|
||||
# Root-level project: "/"
|
||||
# Fullstack project: "/backend"
|
||||
|
||||
- package-ecosystem: "pip"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
day: "monday"
|
||||
# Groups batch related updates into fewer PRs.
|
||||
# Without groups, each dep gets its own PR — noisy for active projects.
|
||||
groups:
|
||||
python-deps:
|
||||
patterns: ["*"]
|
||||
labels:
|
||||
- "dependencies"
|
||||
- "python"
|
||||
commit-message:
|
||||
prefix: "chore(deps)"
|
||||
|
||||
|
||||
# ── npm / pnpm ────────────────────────────────────────────────────────────
|
||||
# Covers package.json. Works with pnpm lockfiles.
|
||||
# Delete this section for Python-only projects.
|
||||
|
||||
- package-ecosystem: "npm"
|
||||
directory: "/frontend"
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
day: "monday"
|
||||
groups:
|
||||
# Split prod and dev deps so you can prioritize runtime updates.
|
||||
npm-prod:
|
||||
dependency-type: "production"
|
||||
patterns: ["*"]
|
||||
npm-dev:
|
||||
dependency-type: "development"
|
||||
patterns: ["*"]
|
||||
labels:
|
||||
- "dependencies"
|
||||
- "javascript"
|
||||
commit-message:
|
||||
prefix: "chore(deps)"
|
||||
|
||||
|
||||
# ── Docker ────────────────────────────────────────────────────────────────
|
||||
# Updates base image tags in Dockerfiles (e.g., python:3.13-slim).
|
||||
# Delete this section if your project has no Dockerfiles.
|
||||
|
||||
- package-ecosystem: "docker"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
labels:
|
||||
- "dependencies"
|
||||
- "docker"
|
||||
commit-message:
|
||||
prefix: "chore(deps)"
|
||||
|
||||
|
||||
# ── GitHub Actions ────────────────────────────────────────────────────────
|
||||
# Updates action versions in .github/workflows/*.yml.
|
||||
# Pinned actions (e.g., actions/checkout@v4) can fall behind on patches.
|
||||
# This keeps them current.
|
||||
|
||||
- package-ecosystem: "github-actions"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
labels:
|
||||
- "dependencies"
|
||||
- "ci"
|
||||
commit-message:
|
||||
prefix: "chore(ci)"
|
||||
|
||||
|
||||
# ── Go (uncomment for Go projects) ────────────────────────────────────────
|
||||
# Updates go.mod dependencies.
|
||||
#
|
||||
# - package-ecosystem: "gomod"
|
||||
# directory: "/"
|
||||
# schedule:
|
||||
# interval: "weekly"
|
||||
# groups:
|
||||
# go-deps:
|
||||
# patterns: ["*"]
|
||||
# labels:
|
||||
# - "dependencies"
|
||||
# - "go"
|
||||
# commit-message:
|
||||
# prefix: "chore(deps)"
|
||||
|
|
@ -0,0 +1,384 @@
|
|||
# ©AngelaMos | 2026
|
||||
# justfile.example
|
||||
#
|
||||
# Annotated justfile template for Cybersecurity-Projects.
|
||||
# Copy into your project root, rename to `justfile`, and replace
|
||||
# every <PLACEHOLDER> with your actual values. Delete sections
|
||||
# that don't apply to your project's stack.
|
||||
#
|
||||
# Docs: https://just.systems/man/en/
|
||||
# Cheat: https://cheatography.com/linux-china/cheat-sheets/justfile/
|
||||
#
|
||||
# Install just:
|
||||
# cargo install just (from source)
|
||||
# brew install just (macOS)
|
||||
# apt install just (Debian/Ubuntu 24.04+)
|
||||
# pipx install rust-just (via pipx)
|
||||
#
|
||||
# Usage:
|
||||
# just list all available recipes
|
||||
# just <recipe> run a recipe
|
||||
# just -n <recipe> dry-run (print commands without executing)
|
||||
|
||||
|
||||
# ── Settings ──────────────────────────────────────────────────────────────────
|
||||
# These apply globally to every recipe in the file.
|
||||
|
||||
# Load .env file into environment for all recipes.
|
||||
set dotenv-load
|
||||
|
||||
# Export all `just` variables as environment variables to child processes.
|
||||
set export
|
||||
|
||||
# Use bash in strict mode (-u = error on unset vars, -c = command string).
|
||||
# This gives consistent behavior across macOS/Linux regardless of login shell.
|
||||
set shell := ["bash", "-uc"]
|
||||
|
||||
# Windows equivalent (only takes effect on Windows, ignored elsewhere).
|
||||
set windows-shell := ["powershell.exe", "-NoLogo", "-Command"]
|
||||
|
||||
|
||||
# ── Variables ─────────────────────────────────────────────────────────────────
|
||||
# Variables defined here are available in all recipes via {{name}}.
|
||||
# Backtick expressions run shell commands at parse time.
|
||||
|
||||
project := file_name(justfile_directory())
|
||||
version := `git describe --tags --always 2>/dev/null || echo "dev"`
|
||||
|
||||
# Project-specific paths — change these to match your layout.
|
||||
# Python-only project: src_dir := "src"
|
||||
# Fullstack project: src_dir := "backend/src"
|
||||
src_dir := "<src>"
|
||||
test_dir := "<tests>"
|
||||
package := "<package-name>"
|
||||
|
||||
|
||||
# ── Default ───────────────────────────────────────────────────────────────────
|
||||
# The first recipe is the default (runs when you type `just` with no args).
|
||||
# --unsorted preserves the order defined in this file instead of alphabetizing.
|
||||
|
||||
default:
|
||||
@just --list --unsorted
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Linting and Formatting
|
||||
# =============================================================================
|
||||
# Ruff handles both linting and formatting for Python.
|
||||
# `*ARGS` is a variadic parameter — any extra flags you pass get forwarded.
|
||||
# Example: just ruff --fix
|
||||
|
||||
[group('lint')]
|
||||
ruff *ARGS:
|
||||
ruff check {{src_dir}}/ {{test_dir}}/ {{ARGS}}
|
||||
|
||||
[group('lint')]
|
||||
ruff-fix:
|
||||
ruff check {{src_dir}}/ {{test_dir}}/ --fix
|
||||
ruff format {{src_dir}}/ {{test_dir}}/
|
||||
|
||||
[group('lint')]
|
||||
ruff-format:
|
||||
ruff format {{src_dir}}/ {{test_dir}}/
|
||||
|
||||
[group('lint')]
|
||||
pylint *ARGS:
|
||||
pylint {{src_dir}}/{{package}} {{ARGS}}
|
||||
|
||||
# Dependency recipes — `lint` runs `ruff` then `pylint` in order.
|
||||
[group('lint')]
|
||||
lint: ruff pylint
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Frontend Linting (delete this section for Python-only projects)
|
||||
# =============================================================================
|
||||
# Biome is an all-in-one linter/formatter for JS/TS (replaces ESLint+Prettier).
|
||||
# Alternatives: eslint + prettier (shown commented below)
|
||||
|
||||
[group('frontend')]
|
||||
biome *ARGS:
|
||||
cd frontend && pnpm biome check . {{ARGS}}
|
||||
|
||||
[group('frontend')]
|
||||
biome-fix:
|
||||
cd frontend && pnpm biome check --write .
|
||||
|
||||
[group('frontend')]
|
||||
stylelint *ARGS:
|
||||
cd frontend && pnpm stylelint '**/*.scss' {{ARGS}}
|
||||
|
||||
[group('frontend')]
|
||||
stylelint-fix:
|
||||
cd frontend && pnpm stylelint '**/*.scss' --fix
|
||||
|
||||
[group('frontend')]
|
||||
tsc *ARGS:
|
||||
cd frontend && pnpm tsc --noEmit {{ARGS}}
|
||||
|
||||
# Alternative: ESLint + Prettier (uncomment if not using Biome)
|
||||
# [group('frontend')]
|
||||
# eslint *ARGS:
|
||||
# cd frontend && pnpm eslint . {{ARGS}}
|
||||
#
|
||||
# [group('frontend')]
|
||||
# eslint-fix:
|
||||
# cd frontend && pnpm eslint . --fix
|
||||
#
|
||||
# [group('frontend')]
|
||||
# prettier *ARGS:
|
||||
# cd frontend && pnpm prettier --check "src/**/*.{ts,tsx,css}" {{ARGS}}
|
||||
#
|
||||
# [group('frontend')]
|
||||
# prettier-fix:
|
||||
# cd frontend && pnpm prettier --write "src/**/*.{ts,tsx,css}"
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Type Checking
|
||||
# =============================================================================
|
||||
|
||||
[group('types')]
|
||||
mypy *ARGS:
|
||||
mypy {{src_dir}}/ {{ARGS}}
|
||||
|
||||
# ty is Astral's experimental type checker (fast, Rust-based).
|
||||
# Uncomment if your project uses it.
|
||||
# [group('types')]
|
||||
# ty *ARGS:
|
||||
# ty check {{ARGS}}
|
||||
|
||||
[group('types')]
|
||||
typecheck: mypy
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Testing
|
||||
# =============================================================================
|
||||
|
||||
[group('test')]
|
||||
pytest *ARGS:
|
||||
pytest {{test_dir}}/ {{ARGS}}
|
||||
|
||||
[group('test')]
|
||||
test: pytest
|
||||
|
||||
[group('test')]
|
||||
test-cov:
|
||||
pytest {{test_dir}}/ --cov={{src_dir}}/{{package}} --cov-report=term-missing --cov-report=html
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# CI / Quality
|
||||
# =============================================================================
|
||||
# Composite recipes that chain multiple checks.
|
||||
# `ci` is the full pipeline; `check` is the fast subset (no tests).
|
||||
|
||||
[group('ci')]
|
||||
ci: lint typecheck test
|
||||
|
||||
[group('ci')]
|
||||
check: ruff mypy
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Docker Compose — Production (delete if no Docker)
|
||||
# =============================================================================
|
||||
# These wrap `docker compose` (compose.yml).
|
||||
# Default params use `service='backend'` syntax — overridable at call site.
|
||||
|
||||
[group('docker')]
|
||||
up *ARGS:
|
||||
docker compose up {{ARGS}}
|
||||
|
||||
[group('docker')]
|
||||
start *ARGS:
|
||||
docker compose up -d {{ARGS}}
|
||||
|
||||
[group('docker')]
|
||||
down *ARGS:
|
||||
docker compose down {{ARGS}}
|
||||
|
||||
[group('docker')]
|
||||
stop:
|
||||
docker compose stop
|
||||
|
||||
[group('docker')]
|
||||
build *ARGS:
|
||||
docker compose build {{ARGS}}
|
||||
|
||||
[group('docker')]
|
||||
rebuild:
|
||||
docker compose build --no-cache
|
||||
|
||||
[group('docker')]
|
||||
logs *SERVICE:
|
||||
docker compose logs -f {{SERVICE}}
|
||||
|
||||
[group('docker')]
|
||||
ps:
|
||||
docker compose ps
|
||||
|
||||
[group('docker')]
|
||||
shell service='backend':
|
||||
docker compose exec -it {{service}} /bin/bash
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Docker Compose — Dev (delete if no Docker)
|
||||
# =============================================================================
|
||||
# Same commands but targeting dev.compose.yml.
|
||||
|
||||
[group('dev')]
|
||||
dev-up *ARGS:
|
||||
docker compose -f dev.compose.yml up {{ARGS}}
|
||||
|
||||
[group('dev')]
|
||||
dev-start *ARGS:
|
||||
docker compose -f dev.compose.yml up -d {{ARGS}}
|
||||
|
||||
[group('dev')]
|
||||
dev-down *ARGS:
|
||||
docker compose -f dev.compose.yml down {{ARGS}}
|
||||
|
||||
[group('dev')]
|
||||
dev-stop:
|
||||
docker compose -f dev.compose.yml stop
|
||||
|
||||
[group('dev')]
|
||||
dev-build *ARGS:
|
||||
docker compose -f dev.compose.yml build {{ARGS}}
|
||||
|
||||
[group('dev')]
|
||||
dev-rebuild:
|
||||
docker compose -f dev.compose.yml build --no-cache
|
||||
|
||||
[group('dev')]
|
||||
dev-logs *SERVICE:
|
||||
docker compose -f dev.compose.yml logs -f {{SERVICE}}
|
||||
|
||||
[group('dev')]
|
||||
dev-ps:
|
||||
docker compose -f dev.compose.yml ps
|
||||
|
||||
[group('dev')]
|
||||
dev-shell service='backend':
|
||||
docker compose -f dev.compose.yml exec -it {{service}} /bin/bash
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Database — Docker (delete if no database)
|
||||
# =============================================================================
|
||||
# Alembic migrations executed inside the running backend container.
|
||||
|
||||
[group('db')]
|
||||
migrate *ARGS:
|
||||
docker compose exec backend alembic upgrade {{ARGS}}
|
||||
|
||||
[group('db')]
|
||||
migration message:
|
||||
docker compose exec backend alembic revision --autogenerate -m "{{message}}"
|
||||
|
||||
[group('db')]
|
||||
rollback:
|
||||
docker compose exec backend alembic downgrade -1
|
||||
|
||||
[group('db')]
|
||||
db-history:
|
||||
docker compose exec backend alembic history --verbose
|
||||
|
||||
[group('db')]
|
||||
db-current:
|
||||
docker compose exec backend alembic current
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Database — Local (no Docker) (delete if no database)
|
||||
# =============================================================================
|
||||
|
||||
[group('db-local')]
|
||||
migrate-local *ARGS:
|
||||
cd backend && uv run alembic upgrade {{ARGS}}
|
||||
|
||||
[group('db-local')]
|
||||
migration-local message:
|
||||
cd backend && uv run alembic revision --autogenerate -m "{{message}}"
|
||||
|
||||
[group('db-local')]
|
||||
rollback-local:
|
||||
cd backend && uv run alembic downgrade -1
|
||||
|
||||
[group('db-local')]
|
||||
db-history-local:
|
||||
cd backend && uv run alembic history --verbose
|
||||
|
||||
[group('db-local')]
|
||||
db-current-local:
|
||||
cd backend && uv run alembic current
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Setup
|
||||
# =============================================================================
|
||||
# First-time project bootstrap. For fullstack projects, chain sub-recipes.
|
||||
|
||||
[group('setup')]
|
||||
setup:
|
||||
@echo "Setting up development environment..."
|
||||
uv sync --all-extras
|
||||
@echo "Setup complete!"
|
||||
|
||||
# Fullstack alternative (uncomment and replace the simple setup above):
|
||||
# [group('setup')]
|
||||
# setup: setup-backend setup-frontend env
|
||||
# @echo "Setup complete!"
|
||||
#
|
||||
# [group('setup')]
|
||||
# setup-backend:
|
||||
# @echo "Setting up backend..."
|
||||
# cd backend && uv sync
|
||||
# @echo "Backend ready!"
|
||||
#
|
||||
# [group('setup')]
|
||||
# setup-frontend:
|
||||
# @echo "Setting up frontend..."
|
||||
# cd frontend && pnpm install
|
||||
# @echo "Frontend ready!"
|
||||
#
|
||||
# [group('setup')]
|
||||
# env:
|
||||
# @if [ ! -f .env ]; then cp .env.example .env; echo "Created .env from .env.example"; fi
|
||||
# @echo ".env ready — update with your values."
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Utilities
|
||||
# =============================================================================
|
||||
|
||||
[group('util')]
|
||||
info:
|
||||
@echo "Project: {{project}}"
|
||||
@echo "Version: {{version}}"
|
||||
@echo "OS: {{os()}} ({{arch()}})"
|
||||
|
||||
# The `-` prefix ignores errors (e.g., dir doesn't exist). Keeps going.
|
||||
[group('util')]
|
||||
clean:
|
||||
-rm -rf .mypy_cache
|
||||
-rm -rf .pytest_cache
|
||||
-rm -rf .ruff_cache
|
||||
-rm -rf htmlcov
|
||||
-rm -f .coverage
|
||||
-rm -rf dist
|
||||
-rm -rf build
|
||||
@echo "Cache directories cleaned"
|
||||
|
||||
# [confirm(...)] prompts for Y/n before running — safety net for destructive ops.
|
||||
[group('util')]
|
||||
[confirm("Remove all build artifacts, caches, and virtual environment?")]
|
||||
nuke:
|
||||
@echo "Nuking everything..."
|
||||
-rm -rf .mypy_cache .pytest_cache .ruff_cache htmlcov dist build
|
||||
-rm -f .coverage
|
||||
-rm -rf .venv
|
||||
@echo "Nuke complete!"
|
||||
|
|
@ -0,0 +1,106 @@
|
|||
{
|
||||
"//": [
|
||||
"©AngelaMos | 2026",
|
||||
"package.json.example",
|
||||
"",
|
||||
"Annotated package.json template for Cybersecurity-Projects frontends.",
|
||||
"Copy into your frontend/ directory, rename to package.json, and replace",
|
||||
"every <PLACEHOLDER> with your actual values.",
|
||||
"",
|
||||
"JSON has no comment syntax, so '//' keys are the standard convention",
|
||||
"for documentation in package.json. They are ignored by pnpm/npm.",
|
||||
"",
|
||||
"Docs: https://docs.npmjs.com/cli/v10/configuring-npm/package-json",
|
||||
"pnpm: https://pnpm.io/package_json"
|
||||
],
|
||||
|
||||
"name": "<project-name>",
|
||||
"private": true,
|
||||
"version": "1.0.0",
|
||||
"description": "<One-line description of the frontend>",
|
||||
"type": "module",
|
||||
|
||||
"// scripts": [
|
||||
"Scripts run via `pnpm <script>` (no `run` needed for lifecycle scripts).",
|
||||
"Keep names consistent across all projects so the root justfile works.",
|
||||
"",
|
||||
"Naming convention:",
|
||||
" verb dev, build, preview, test",
|
||||
" verb:qualifier lint:fix, lint:scss, format:check",
|
||||
"",
|
||||
"pnpm scripts docs: https://docs.npmjs.com/cli/v10/using-npm/scripts"
|
||||
],
|
||||
"scripts": {
|
||||
"dev": "vite",
|
||||
"build": "tsc -b && vite build",
|
||||
"preview": "vite preview",
|
||||
|
||||
"lint": "biome check .",
|
||||
"lint:fix": "biome check --write .",
|
||||
"format": "biome format --write .",
|
||||
|
||||
"typecheck": "tsc -b",
|
||||
|
||||
"lint:scss": "stylelint '**/*.scss'",
|
||||
"lint:scss:fix": "stylelint '**/*.scss' --fix"
|
||||
},
|
||||
|
||||
"// dependencies": [
|
||||
"Runtime deps — these ship in the production bundle.",
|
||||
"Only add what you actually import. Transitive deps are pulled automatically."
|
||||
],
|
||||
"dependencies": {
|
||||
"@tanstack/react-query": "^5.90.12",
|
||||
"axios": "^1.13.0",
|
||||
"react": "^19.2.1",
|
||||
"react-dom": "^19.2.0",
|
||||
"react-error-boundary": "^6.0.0",
|
||||
"react-icons": "^5.5.0",
|
||||
"react-router-dom": "^7.1.1",
|
||||
"sonner": "^2.0.7",
|
||||
"zod": "^4.1.13",
|
||||
"zustand": "^5.0.9"
|
||||
},
|
||||
|
||||
"// devDependencies": [
|
||||
"Dev deps — only used during development/build, not shipped to users.",
|
||||
"Linters, type checkers, build tools, test frameworks go here."
|
||||
],
|
||||
"devDependencies": {
|
||||
"@biomejs/biome": "^2.4.2",
|
||||
"@tanstack/react-query-devtools": "^5.91.1",
|
||||
"@types/node": "^24.10.2",
|
||||
"@types/react": "^19.2.7",
|
||||
"@types/react-dom": "^19.2.3",
|
||||
"@vitejs/plugin-react": "^5.1.1",
|
||||
"sass": "^1.95.0",
|
||||
"stylelint": "^16.26.1",
|
||||
"stylelint-config-prettier-scss": "^1.0.0",
|
||||
"stylelint-config-standard-scss": "^16.0.0",
|
||||
"typescript": "~5.9.3",
|
||||
"vite": "npm:rolldown-vite@7.2.5",
|
||||
"vite-tsconfig-paths": "^5.1.0"
|
||||
},
|
||||
|
||||
"// pnpm": [
|
||||
"pnpm-specific configuration.",
|
||||
"",
|
||||
"overrides: Force a specific version of a transitive dependency.",
|
||||
"Use this when a dep pulls in a version you don't want.",
|
||||
"Example: pin vite to rolldown-vite for faster builds.",
|
||||
"",
|
||||
"peerDependencyRules.allowAny: Suppress peer dep warnings for",
|
||||
"packages that haven't updated their peerDeps to React 19 yet."
|
||||
],
|
||||
"pnpm": {
|
||||
"overrides": {
|
||||
"vite": "npm:rolldown-vite@7.2.5"
|
||||
},
|
||||
"peerDependencyRules": {
|
||||
"allowAny": [
|
||||
"react",
|
||||
"react-dom"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,246 @@
|
|||
# ©AngelaMos | 2026
|
||||
# pyproject.toml.example
|
||||
#
|
||||
# Annotated pyproject.toml template for Cybersecurity-Projects.
|
||||
# Copy this file into your project root, rename to pyproject.toml,
|
||||
# and replace every <PLACEHOLDER> with your actual values.
|
||||
#
|
||||
# Official guide: https://packaging.python.org/en/latest/guides/writing-pyproject-toml/
|
||||
# Full spec: https://packaging.python.org/en/latest/specifications/pyproject-toml/
|
||||
|
||||
|
||||
# ── Build System ──────────────────────────────────────────────────────────────
|
||||
# Declares HOW your project is built. This must appear for the project
|
||||
# to be installable via `uv pip install .` or any PEP 517 frontend.
|
||||
# Hatchling is a lightweight, modern build backend.
|
||||
# Alternatives: setuptools, flit-core, pdm-backend, maturin (Rust extensions)
|
||||
|
||||
[build-system]
|
||||
requires = ["hatchling"]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
|
||||
# ── Project Metadata ──────────────────────────────────────────────────────────
|
||||
# Core metadata that identifies your package.
|
||||
# Spec: https://packaging.python.org/en/latest/specifications/core-metadata/
|
||||
|
||||
[project]
|
||||
name = "<project-name>"
|
||||
version = "<x.x.x>"
|
||||
description = "<One-line description of what this project does>"
|
||||
readme = "README.md"
|
||||
license = { text = "MIT" }
|
||||
requires-python = ">=3.13"
|
||||
|
||||
# Keywords help with discoverability on PyPI.
|
||||
# Pick 3-6 terms someone would search for.
|
||||
keywords = ["security", "<keyword>", "<keyword>"]
|
||||
|
||||
# Classifiers are a fixed vocabulary — pick from:
|
||||
# https://pypi.org/classifiers/
|
||||
classifiers = [
|
||||
"Development Status :: 3 - Alpha",
|
||||
"Intended Audience :: Developers",
|
||||
"License :: OSI Approved :: MIT License",
|
||||
"Operating System :: OS Independent",
|
||||
"Programming Language :: Python :: 3",
|
||||
"Programming Language :: Python :: 3.13",
|
||||
"Programming Language :: Python :: 3.14",
|
||||
"Topic :: Security",
|
||||
"Typing :: Typed",
|
||||
]
|
||||
|
||||
# ── Dependencies ──────────────────────────────────────────────────────────────
|
||||
# Runtime dependencies installed whenever your package is installed.
|
||||
#
|
||||
# Version pinning cheat-sheet:
|
||||
# "pkg>=1.2" 1.2 or newer (most common)
|
||||
# "pkg>=1.2,<2" 1.2+ but not 2.0 (pin major version)
|
||||
# "pkg~=1.2" same as >=1.2,<2.0 (compatible release)
|
||||
# "pkg==1.2.3" exact pin (only for reproducible builds, avoid in libs)
|
||||
#
|
||||
# Only list what you actually import. Transitive deps are pulled automatically.
|
||||
|
||||
dependencies = []
|
||||
|
||||
# ── Optional Dependency Groups ────────────────────────────────────────────────
|
||||
# Install with: uv pip install ".[dev]" or uv pip install ".[dev,docs]"
|
||||
# Keep groups focused — one concern per group.
|
||||
|
||||
[project.optional-dependencies]
|
||||
dev = [
|
||||
"pytest>=9.0.2",
|
||||
"ruff>=0.15.1",
|
||||
]
|
||||
|
||||
# ── URLs ──────────────────────────────────────────────────────────────────────
|
||||
# Shown on PyPI sidebar and by `uv pip show`.
|
||||
|
||||
[project.urls]
|
||||
Homepage = "https://github.com/<user>/<repo>"
|
||||
Repository = "https://github.com/<user>/<repo>"
|
||||
Issues = "https://github.com/<user>/<repo>/issues"
|
||||
|
||||
# ── Entry Points (uncomment if needed) ────────────────────────────────────────
|
||||
# CLI commands:
|
||||
# [project.scripts]
|
||||
# mycli = "<package>.cli:main"
|
||||
#
|
||||
# Plugin hooks (e.g. pytest plugins):
|
||||
# [project.entry-points."pytest11"]
|
||||
# myplugin = "<package>.pytest_plugin"
|
||||
|
||||
|
||||
# ── Hatch Build ───────────────────────────────────────────────────────────────
|
||||
# Tells hatchling where your importable code lives.
|
||||
# Adjust the path to match your project layout.
|
||||
#
|
||||
# src layout → packages = ["src/<package>"]
|
||||
# flat layout → packages = ["<package>"]
|
||||
|
||||
[tool.hatch.build.targets.wheel]
|
||||
packages = ["src/<package>"]
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
# TOOL CONFIGURATION
|
||||
# Everything below configures dev tools. None of it affects the built package.
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
|
||||
# ── Ruff (linter + formatter) ─────────────────────────────────────────────────
|
||||
# Docs: https://docs.astral.sh/ruff/configuration/
|
||||
# Rules: https://docs.astral.sh/ruff/rules/
|
||||
|
||||
[tool.ruff]
|
||||
target-version = "py313"
|
||||
line-length = 88
|
||||
src = ["src"]
|
||||
|
||||
[tool.ruff.lint]
|
||||
select = [
|
||||
"E", # pycodestyle errors
|
||||
"W", # pycodestyle warnings
|
||||
"F", # pyflakes
|
||||
"B", # flake8-bugbear
|
||||
"C4", # flake8-comprehensions
|
||||
"UP", # pyupgrade
|
||||
"ARG", # flake8-unused-arguments
|
||||
"SIM", # flake8-simplify
|
||||
"PTH", # flake8-use-pathlib
|
||||
"RUF", # ruff-specific rules
|
||||
"ASYNC", # flake8-async
|
||||
"S", # flake8-bandit (security)
|
||||
"N", # pep8-naming
|
||||
]
|
||||
|
||||
# Add project-specific ignores here. Some common ones:
|
||||
# "E501" — line too long (let the formatter handle it)
|
||||
# "B008" — function call in default arg (FastAPI Depends pattern)
|
||||
# "S101" — assert usage (needed in tests)
|
||||
# "ARG001" — unused function arg (common in FastAPI deps / callbacks)
|
||||
ignore = [
|
||||
"E501",
|
||||
]
|
||||
|
||||
[tool.ruff.lint.per-file-ignores]
|
||||
# Example: allow asserts in tests, ignore security warnings in test fixtures
|
||||
# "tests/**" = ["S101", "S106", "ARG001"]
|
||||
|
||||
|
||||
# ── mypy (static type checker) ────────────────────────────────────────────────
|
||||
# Docs: https://mypy.readthedocs.io/en/stable/config_file.html
|
||||
|
||||
[tool.mypy]
|
||||
python_version = "3.13"
|
||||
strict = true
|
||||
warn_return_any = true
|
||||
warn_unused_ignores = true
|
||||
disallow_untyped_defs = true
|
||||
disallow_incomplete_defs = true
|
||||
exclude = [".venv", "venv"]
|
||||
|
||||
# Per-module overrides for third-party libs that lack type stubs:
|
||||
# [[tool.mypy.overrides]]
|
||||
# module = ["some_untyped_lib", "another_lib.*"]
|
||||
# ignore_missing_imports = true
|
||||
|
||||
|
||||
# ── pylint ────────────────────────────────────────────────────────────────────
|
||||
# Docs: https://pylint.readthedocs.io/en/stable/user_guide/configuration/index.html
|
||||
|
||||
[tool.pylint.main]
|
||||
py-version = "3.13"
|
||||
jobs = 4
|
||||
persistent = true
|
||||
ignore = [
|
||||
"venv",
|
||||
".venv",
|
||||
"__pycache__",
|
||||
"build",
|
||||
"dist",
|
||||
".git",
|
||||
]
|
||||
|
||||
[tool.pylint.messages_control]
|
||||
# Start minimal — only disable what your project actually hits.
|
||||
# Run `pylint src/` and decide per-code whether to fix or disable.
|
||||
disable = [
|
||||
"C0301", # line-too-long (ruff formatter handles this)
|
||||
]
|
||||
|
||||
[tool.pylint.format]
|
||||
max-line-length = 95
|
||||
|
||||
[tool.pylint.design]
|
||||
max-args = 10
|
||||
max-attributes = 10
|
||||
|
||||
|
||||
# ── pytest ────────────────────────────────────────────────────────────────────
|
||||
# Docs: https://docs.pytest.org/en/stable/reference/customize.html
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
testpaths = ["tests"]
|
||||
addopts = "-ra -q"
|
||||
filterwarnings = [
|
||||
"ignore::DeprecationWarning",
|
||||
]
|
||||
|
||||
# For async tests (uncomment if using pytest-asyncio):
|
||||
# asyncio_mode = "auto"
|
||||
# asyncio_default_fixture_loop_scope = "function"
|
||||
|
||||
|
||||
# ── coverage ──────────────────────────────────────────────────────────────────
|
||||
# Docs: https://coverage.readthedocs.io/en/latest/config.html
|
||||
# Run: pytest --cov=src --cov-report=term-missing
|
||||
|
||||
[tool.coverage.run]
|
||||
branch = true
|
||||
source = ["src"]
|
||||
|
||||
[tool.coverage.report]
|
||||
exclude_lines = [
|
||||
"pragma: no cover",
|
||||
"if TYPE_CHECKING:",
|
||||
"raise NotImplementedError",
|
||||
]
|
||||
|
||||
|
||||
# ── ty (red-knot type checker, experimental) ──────────────────────────────────
|
||||
# Docs: https://docs.astral.sh/ty/configuration/
|
||||
# Uncomment if your project uses ty.
|
||||
#
|
||||
# [tool.ty.src]
|
||||
# include = ["src", "tests"]
|
||||
# exclude = [".venv/**"]
|
||||
#
|
||||
# [tool.ty.environment]
|
||||
# python-version = "3.13"
|
||||
# root = ["./src"]
|
||||
# python = "./.venv"
|
||||
#
|
||||
# [tool.ty.rules]
|
||||
# possibly-missing-attribute = "error"
|
||||
# possibly-missing-import = "error"
|
||||
|
|
@ -0,0 +1,73 @@
|
|||
{
|
||||
"//": [
|
||||
"©AngelaMos | 2026",
|
||||
"tsconfig.app.json.example",
|
||||
"",
|
||||
"TypeScript config for BROWSER code (everything in src/).",
|
||||
"This is where React components, hooks, and pages live.",
|
||||
"",
|
||||
"Docs: https://www.typescriptlang.org/tsconfig/"
|
||||
],
|
||||
|
||||
"compilerOptions": {
|
||||
"// tsBuildInfoFile": "Cache for incremental builds — speeds up `tsc -b` significantly.",
|
||||
"tsBuildInfoFile": "./node_modules/.tmp/tsconfig.app.tsbuildinfo",
|
||||
|
||||
"// target": "ES2022 is safe for all modern browsers. Gives you top-level await, class fields, etc.",
|
||||
"target": "ES2022",
|
||||
|
||||
"useDefineForClassFields": true,
|
||||
|
||||
"// lib": "ES2022 for JS features, DOM + DOM.Iterable for browser APIs (document, fetch, etc.).",
|
||||
"lib": ["ES2022", "DOM", "DOM.Iterable"],
|
||||
|
||||
"// module": "ESNext tells TS to output modern import/export. Vite handles bundling from there.",
|
||||
"module": "ESNext",
|
||||
|
||||
"// types": "Only include Vite's client types (import.meta.env, asset imports, etc.).",
|
||||
"types": ["vite/client"],
|
||||
|
||||
"skipLibCheck": true,
|
||||
|
||||
"// moduleResolution": [
|
||||
"'bundler' is the modern setting for Vite/webpack/esbuild projects.",
|
||||
"It understands package.json 'exports' and doesn't require file extensions.",
|
||||
"Do NOT use 'node' or 'node16' — those are for Node.js, not bundlers."
|
||||
],
|
||||
"moduleResolution": "bundler",
|
||||
"allowImportingTsExtensions": true,
|
||||
|
||||
"// verbatimModuleSyntax": "Forces `import type { Foo }` syntax. Helps bundlers tree-shake better.",
|
||||
"verbatimModuleSyntax": true,
|
||||
|
||||
"moduleDetection": "force",
|
||||
"noEmit": true,
|
||||
|
||||
"// jsx": "'react-jsx' uses the automatic JSX transform (React 17+). No need to import React.",
|
||||
"jsx": "react-jsx",
|
||||
|
||||
"// strict options": [
|
||||
"strict: true enables ALL strict checks as a baseline.",
|
||||
"The individual flags below go BEYOND strict for maximum safety.",
|
||||
"noUncheckedSideEffectImports catches `import './styles.css'` typos."
|
||||
],
|
||||
"strict": true,
|
||||
"noUnusedLocals": true,
|
||||
"noUnusedParameters": true,
|
||||
"erasableSyntaxOnly": true,
|
||||
"noFallthroughCasesInSwitch": true,
|
||||
"noUncheckedSideEffectImports": true,
|
||||
|
||||
"// paths": [
|
||||
"Path aliases so you write `import { Button } from '@/components/Button'`",
|
||||
"instead of `import { Button } from '../../../components/Button'`.",
|
||||
"Requires vite-tsconfig-paths plugin in vite.config.ts to work at runtime."
|
||||
],
|
||||
"baseUrl": ".",
|
||||
"paths": {
|
||||
"@/*": ["./src/*"]
|
||||
}
|
||||
},
|
||||
|
||||
"include": ["src"]
|
||||
}
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
{
|
||||
"//": [
|
||||
"©AngelaMos | 2026",
|
||||
"tsconfig.json.example",
|
||||
"",
|
||||
"Annotated TypeScript configuration for Cybersecurity-Projects frontends.",
|
||||
"This is a 3-file setup used by Vite + React projects:",
|
||||
"",
|
||||
" tsconfig.json Root — references the other two (this file)",
|
||||
" tsconfig.app.json Browser code (src/) — React, DOM APIs",
|
||||
" tsconfig.node.json Node code (vite.config.ts) — Node APIs",
|
||||
"",
|
||||
"Why split? Browser code and Node code have different globals (window vs",
|
||||
"process), different module systems, and different type libraries. Splitting",
|
||||
"prevents accidental use of Node APIs in browser code and vice versa.",
|
||||
"",
|
||||
"Copy all three files into your frontend/ directory, remove the .example",
|
||||
"suffix from this file, and use the other two as-is.",
|
||||
"",
|
||||
"Docs: https://www.typescriptlang.org/tsconfig/"
|
||||
],
|
||||
|
||||
"// files": "Empty array means this file compiles nothing directly — it only references sub-configs.",
|
||||
"files": [],
|
||||
"references": [
|
||||
{ "path": "./tsconfig.app.json" },
|
||||
{ "path": "./tsconfig.node.json" }
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
{
|
||||
"//": [
|
||||
"©AngelaMos | 2026",
|
||||
"tsconfig.node.json.example",
|
||||
"",
|
||||
"TypeScript config for NODE code (vite.config.ts and other build scripts).",
|
||||
"This runs in Node.js during build, NOT in the browser.",
|
||||
"",
|
||||
"Docs: https://www.typescriptlang.org/tsconfig/"
|
||||
],
|
||||
|
||||
"compilerOptions": {
|
||||
"tsBuildInfoFile": "./node_modules/.tmp/tsconfig.node.tsbuildinfo",
|
||||
|
||||
"// target": "ES2023 — Node 20+ supports everything in ES2023.",
|
||||
"target": "ES2023",
|
||||
|
||||
"// lib": "ES2023 only — no DOM here since this runs in Node, not a browser.",
|
||||
"lib": ["ES2023"],
|
||||
|
||||
"module": "ESNext",
|
||||
|
||||
"// types": "Only Node types — gives you process, __dirname, Buffer, etc.",
|
||||
"types": ["node"],
|
||||
|
||||
"skipLibCheck": true,
|
||||
"moduleResolution": "bundler",
|
||||
"allowImportingTsExtensions": true,
|
||||
"verbatimModuleSyntax": true,
|
||||
"moduleDetection": "force",
|
||||
"noEmit": true,
|
||||
|
||||
"strict": true,
|
||||
"noUnusedLocals": true,
|
||||
"noUnusedParameters": true,
|
||||
"erasableSyntaxOnly": true,
|
||||
"noFallthroughCasesInSwitch": true,
|
||||
"noUncheckedSideEffectImports": true
|
||||
},
|
||||
|
||||
"// include": "Only the Vite config. Add other Node-side files here if needed (e.g., scripts/).",
|
||||
"include": ["vite.config.ts"]
|
||||
}
|
||||
|
|
@ -1,334 +0,0 @@
|
|||
# Portia — Secrets Scanner Design Document
|
||||
|
||||
**Date**: 2026-02-20
|
||||
**Project**: Cybersecurity-Projects / intermediate / secrets-scanner
|
||||
**Replaces**: Container Security Scanner (overlap with Docker Security Audit)
|
||||
**Module**: `github.com/CarterPerez-dev/portia`
|
||||
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
Portia is a Go CLI tool that scans git repositories and directories for leaked API keys, passwords, tokens, and other secrets. It combines keyword pre-filtering, regex pattern matching, Shannon entropy analysis, and Have I Been Pwned breach verification into a pipeline architecture modeled after production scanners like Gitleaks and TruffleHog.
|
||||
|
||||
**Scan targets**: Git history (via go-git in-process) and filesystem directories.
|
||||
**Not in scope**: Docker image layer extraction (covered by the existing Docker Security Audit project).
|
||||
|
||||
---
|
||||
|
||||
## CLI Structure
|
||||
|
||||
### Commands
|
||||
|
||||
```
|
||||
portia scan [path] Scan a directory/file for secrets
|
||||
portia git [path] Scan git history for secrets
|
||||
portia init Generate .portia.toml config
|
||||
portia config rules List all built-in detection rules
|
||||
portia config test <rule> Test a specific rule against sample input
|
||||
```
|
||||
|
||||
### Global Flags
|
||||
|
||||
```
|
||||
--format string Output format: terminal (default), json, sarif
|
||||
--rules string Path to custom rules TOML file
|
||||
--verbose/-v Show full finding details (line context, entropy scores)
|
||||
--no-color Disable colored output
|
||||
--exit-code int Exit code when secrets found (default: 1, 0 for CI soft-fail)
|
||||
```
|
||||
|
||||
### Scan Flags
|
||||
|
||||
```
|
||||
portia scan:
|
||||
--exclude []string Glob patterns to skip (e.g., "vendor/**", "*.lock")
|
||||
--max-size string Skip files larger than (default: "1MB")
|
||||
--entropy float Override entropy threshold (default: per-rule)
|
||||
--hibp Enable HIBP password breach checking
|
||||
|
||||
portia git:
|
||||
--since string Only scan commits after date (e.g., "7 days ago")
|
||||
--branch string Scan specific branch (default: all)
|
||||
--depth int Max commit depth (default: unlimited)
|
||||
--staged-only Only scan currently staged changes (pre-commit hook mode)
|
||||
--hibp Enable HIBP password breach checking
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Pipeline Architecture
|
||||
|
||||
```
|
||||
Source --> Chunker --> Keyword Filter --> Detector Pool --> Verifier (HIBP) --> Reporter
|
||||
```
|
||||
|
||||
### Stage 1: Source
|
||||
|
||||
Two implementations behind a common interface:
|
||||
- **DirSource**: Walks filesystem, respects .gitignore + .portiaignore, skips binaries and large files.
|
||||
- **GitSource**: Uses go-git to iterate commits, produces diffs (added lines only).
|
||||
|
||||
### Stage 2: Chunker
|
||||
|
||||
Splits source output into Chunk structs containing content, file path, line number, and optional git metadata (commit SHA, author, date).
|
||||
|
||||
### Stage 3: Keyword Filter
|
||||
|
||||
The critical performance optimization. Each rule declares case-insensitive keywords. Before any regex runs, the engine checks whether the chunk contains at least one keyword. This eliminates ~95% of chunks. Only rules whose keywords matched are forwarded to the detector pool for that chunk.
|
||||
|
||||
### Stage 4: Detector Pool
|
||||
|
||||
Bounded concurrency via errgroup.SetLimit(runtime.NumCPU()). For each chunk that passed keyword filtering, matched rules execute:
|
||||
1. Regex match on captured group
|
||||
2. Entropy check on captured value (if rule has a threshold)
|
||||
3. Stopword/allowlist filtering
|
||||
4. Structural validation (assignment operator present, not a template variable)
|
||||
|
||||
### Stage 5: Verifier (HIBP)
|
||||
|
||||
Post-processing enrichment, only when --hibp flag is set. Classifies each finding by secret type, routes passwords to HIBP k-anonymity API, marks results as verified_breach, not_breached, or skipped.
|
||||
|
||||
### Stage 6: Reporter
|
||||
|
||||
Three implementations behind a Reporter interface:
|
||||
- **TerminalReporter**: Angela-style colored output with severity colors, spinners, summary.
|
||||
- **JSONReporter**: Structured JSON to stdout.
|
||||
- **SARIFReporter**: SARIF v2.1.0 for GitHub/GitLab integration.
|
||||
|
||||
### Concurrency Model
|
||||
|
||||
- Channels buffered at 256 to prevent backpressure stalls
|
||||
- errgroup.WithContext for detector pool — first fatal error cancels everything
|
||||
- HIBP lookups run as a separate bounded pool (20 concurrent, 50 req/s rate limit) after scanning completes
|
||||
- Context propagation throughout for clean Ctrl+C cancellation
|
||||
|
||||
---
|
||||
|
||||
## Core Types
|
||||
|
||||
### Finding
|
||||
|
||||
```
|
||||
Finding {
|
||||
RuleID string
|
||||
Description string
|
||||
Severity string // critical, high, medium, low
|
||||
Match string
|
||||
Secret string // The captured secret value (redactable)
|
||||
Entropy float64
|
||||
FilePath string
|
||||
LineNumber int
|
||||
LineContent string
|
||||
CommitSHA string // Empty for dir scan
|
||||
Author string // Empty for dir scan
|
||||
CommitDate time.Time // Empty for dir scan
|
||||
HIBPStatus string // "breached", "clean", "skipped", "unchecked"
|
||||
BreachCount int
|
||||
}
|
||||
```
|
||||
|
||||
### Rule
|
||||
|
||||
```
|
||||
Rule {
|
||||
ID string
|
||||
Description string
|
||||
Severity string
|
||||
Keywords []string
|
||||
Pattern *regexp.Regexp
|
||||
SecretGroup int
|
||||
Entropy *float64 // nil = skip entropy check
|
||||
Allowlist Allowlist
|
||||
SecretType SecretType // Password, APIKey, Token, PrivateKey, etc.
|
||||
}
|
||||
```
|
||||
|
||||
### Chunk
|
||||
|
||||
```
|
||||
Chunk {
|
||||
Content string
|
||||
FilePath string
|
||||
LineStart int
|
||||
CommitSHA string
|
||||
Author string
|
||||
CommitDate time.Time
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Rule Engine
|
||||
|
||||
### 50+ Built-in Rules
|
||||
|
||||
| Category | Count | Examples | Severity |
|
||||
|---|---|---|---|
|
||||
| Cloud providers | ~10 | AWS Access Key, AWS Secret Key, GCP API Key, Azure Client Secret, DigitalOcean PAT | Critical |
|
||||
| Source control | ~8 | GitHub PAT (classic), Fine-grained PAT, OAuth, GitLab PAT, Bitbucket App Password | Critical |
|
||||
| Payment/SaaS | ~8 | Stripe Secret Key, Stripe Restricted Key, SendGrid, Twilio, Shopify | Critical |
|
||||
| Communication | ~5 | Slack Bot/User/App Token, Slack Webhook, Discord Bot Token | High |
|
||||
| AI/ML services | ~4 | OpenAI, Anthropic, HuggingFace, Replicate | High |
|
||||
| Infrastructure | ~5 | Heroku, Terraform Cloud, Vault Token, PlanetScale, Supabase | Critical |
|
||||
| Cryptographic | ~3 | Private Keys (RSA/EC/DSA/OPENSSH), PGP, X.509 | Critical |
|
||||
| Authentication | ~4 | JWT, Bearer Token, Basic Auth, OAuth Client Secret | High |
|
||||
| Database | ~5 | PostgreSQL URI, MySQL URI, MongoDB URI, Redis URI, Generic DB URI | Critical |
|
||||
| Generic | ~3 | Generic API Key (entropy 3.7), Generic Secret (entropy 3.7), Generic Password | Medium |
|
||||
|
||||
### False Positive Defenses (5 Layers)
|
||||
|
||||
**Layer 1 — Keyword pre-filter**: Part of the pipeline. No keyword match = chunk skipped.
|
||||
|
||||
**Layer 2 — Structural validation**: Generic rules require an assignment operator between keyword and value (=, :, =>, :=, ||).
|
||||
|
||||
**Layer 3 — Stopwords**: ~1,500 common programming words. If captured secret contains a stopword, finding is dropped.
|
||||
|
||||
**Layer 4 — Allowlists**:
|
||||
- Global path allowlist: go.sum, package-lock.json, pnpm-lock.yaml, *.min.js, vendor/, node_modules/, .git/, dist-info/
|
||||
- Global value allowlist: placeholder values (EXAMPLE_KEY, your-api-key-here, xxxx, ${VAR}, {{template}}, os.Getenv(...))
|
||||
- Per-rule allowlists for rule-specific exclusions
|
||||
|
||||
**Layer 5 — Entropy validation**: Post-filter on captured secret value.
|
||||
- Base64 charset: 4.5 threshold
|
||||
- Hex charset: 3.0 threshold
|
||||
- Alphanumeric (generic rules): 3.7 threshold
|
||||
- Provider-specific rules with known prefixes: 3.0 threshold
|
||||
|
||||
### TOML Override Format (.portia.toml)
|
||||
|
||||
Users can disable built-in rules, add path excludes, allowlist values, and define custom rules. Custom rules use the same structure as built-in rules (id, description, severity, keywords, pattern, secret-group, entropy).
|
||||
|
||||
---
|
||||
|
||||
## HIBP Integration
|
||||
|
||||
### K-Anonymity Protocol
|
||||
|
||||
1. SHA-1 hash the plaintext password locally
|
||||
2. Send only the 5-character hex prefix to api.pwnedpasswords.com/range/{prefix}
|
||||
3. Receive ~800 suffixes with breach counts
|
||||
4. Match locally — the API never learns the password
|
||||
|
||||
### Architecture
|
||||
|
||||
- **HTTP Client**: hashicorp/go-retryablehttp with custom transport (MaxIdleConnsPerHost: 50, HTTP/2, TLS 1.2+)
|
||||
- **Cache**: hashicorp/golang-lru/v2/expirable — LRU cache keyed by 5-char prefix, 24-hour TTL
|
||||
- **Circuit Breaker**: sony/gobreaker — trips after 5 consecutive failures, 30s recovery timeout
|
||||
- **Rate Limiter**: golang.org/x/time/rate at 50 req/s with burst of 10
|
||||
- **Concurrency**: errgroup.SetLimit(20) for parallel lookups
|
||||
|
||||
### Secret Classification Router
|
||||
|
||||
Only human-chosen passwords are sent to HIBP. Machine-generated secrets (API keys, tokens, private keys) are skipped based on:
|
||||
- Known format prefixes (AKIA, ghp_, sk_live_, xoxb-, SG., sk-)
|
||||
- Keyword context (password, passwd, pwd in surrounding code)
|
||||
- Entropy heuristic (< 3.5 bits + < 32 chars = likely human-chosen)
|
||||
|
||||
### Privacy Safeguards
|
||||
|
||||
- Never transmit full hash or plaintext
|
||||
- Logger set to nil to prevent logging of /range/{prefix} URLs
|
||||
- Results reported as boolean compromised/not-compromised (breach count available in verbose mode only)
|
||||
- HTTPS exclusively, TLS 1.2+ enforced
|
||||
|
||||
---
|
||||
|
||||
## Terminal Output Design
|
||||
|
||||
### Color Mapping
|
||||
|
||||
- CRITICAL: red
|
||||
- HIGH: bold red
|
||||
- MEDIUM: yellow
|
||||
- LOW: cyan
|
||||
- File paths: hi-cyan
|
||||
- Rule IDs: blue
|
||||
- Secret values: dim (partially redacted)
|
||||
- Line numbers: dim italic
|
||||
- HIBP breach warnings: bold yellow
|
||||
|
||||
### Output Sections
|
||||
|
||||
1. Banner (alternating red/blue ASCII art)
|
||||
2. Spinner during scan (cyan braille frame + magenta message)
|
||||
3. Findings grouped by file, sorted by severity
|
||||
4. HIBP breach check section (when --hibp used)
|
||||
5. Summary (files scanned, rules evaluated, secrets found, duration)
|
||||
|
||||
### Redaction
|
||||
|
||||
Secret values are partially redacted by default: first 6 characters shown, rest replaced with "...". Full values shown only with --verbose flag.
|
||||
|
||||
---
|
||||
|
||||
## Project Structure
|
||||
|
||||
```
|
||||
PROJECTS/intermediate/secrets-scanner/
|
||||
cmd/portia/main.go
|
||||
internal/
|
||||
cli/ root.go, scan.go, git.go, init.go, config.go
|
||||
engine/ pipeline.go, chunk.go, detector.go, filter.go
|
||||
source/ source.go, directory.go, git.go
|
||||
rules/ registry.go, builtin.go, custom.go, entropy.go
|
||||
hibp/ client.go, cache.go, breaker.go, classify.go
|
||||
report/ reporter.go, terminal.go, json.go, sarif.go
|
||||
ui/ banner.go, color.go, symbol.go, spinner.go
|
||||
config/ config.go
|
||||
pkg/types/ types.go
|
||||
testdata/ repos/, fixtures/, golden/
|
||||
learn/ 00-OVERVIEW.md through 04-CHALLENGES.md
|
||||
.portia.toml, .golangci.yml, .gitignore, Justfile, go.mod, LICENSE, README.md
|
||||
```
|
||||
|
||||
### Dependencies
|
||||
|
||||
```
|
||||
github.com/spf13/cobra
|
||||
github.com/fatih/color
|
||||
github.com/go-git/go-git/v5
|
||||
github.com/pelletier/go-toml/v2
|
||||
github.com/hashicorp/golang-lru/v2
|
||||
github.com/sony/gobreaker/v2
|
||||
github.com/hashicorp/go-retryablehttp
|
||||
golang.org/x/sync
|
||||
golang.org/x/time
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Testing Strategy
|
||||
|
||||
### Unit Tests
|
||||
|
||||
- **rules/**: Table-driven tests for each of the 50+ rules with true positives, true negatives, and known false positive patterns.
|
||||
- **engine/**: Pipeline stages tested in isolation (chunker, keyword filter, detector, structural validation).
|
||||
- **rules/entropy.go**: Shannon entropy tested against known values from detect-secrets' test suite.
|
||||
- **hibp/**: HTTP test server mocks, cache behavior, circuit breaker states, classification routing.
|
||||
- **source/**: Directory walker respects gitignore, skips binaries, respects max-size. Git source produces correct diffs.
|
||||
- **report/**: JSON and SARIF output compared against golden files.
|
||||
- **config/**: TOML parsing, rule merging, disable/override behavior.
|
||||
|
||||
### Integration Tests
|
||||
|
||||
- testdata/fixtures/ with planted secrets across Python, YAML, .env, JSON, Go, JS files.
|
||||
- testdata/repos/ with small git repos containing commits with secrets added then deleted.
|
||||
- Golden file tests for JSON and SARIF output.
|
||||
|
||||
### CI
|
||||
|
||||
- go test -race ./... (mandatory)
|
||||
- golangci-lint v2 with Tier 1/2/3 linters
|
||||
- gofumpt + golines (80-char max)
|
||||
|
||||
---
|
||||
|
||||
## Learn Folder
|
||||
|
||||
Grounded in three real-world incidents:
|
||||
- **Uber 2022**: Hardcoded HCP Vault credentials in PowerShell script on GitHub
|
||||
- **Samsung 2022**: Leaked SmartThings secrets via accidental git push
|
||||
- **CircleCI 2023**: Rotated all customer secrets after infrastructure breach
|
||||
|
||||
Documentation follows the standard learn/ template: 00-OVERVIEW, 01-CONCEPTS, 02-ARCHITECTURE, 03-IMPLEMENTATION, 04-CHALLENGES.
|
||||
Loading…
Reference in New Issue