Cybersecurity-Projects/PROJECTS/intermediate/secrets-scanner/internal/source/directory.go

// ©AngelaMos | 2026
// directory.go

package source

import (
	"bufio"
	"context"
	"io/fs"
	"os"
	"path/filepath"
	"strings"

	"github.com/CarterPerez-dev/portia/pkg/types"
)

const defaultMaxFileSize = 1 << 20

type Directory struct {
	Path     string
	MaxSize  int64
	Excludes []string
}

func NewDirectory(
	path string, maxSize int64, excludes []string,
) *Directory {
	if maxSize <= 0 {
		maxSize = defaultMaxFileSize
	}
	return &Directory{
		Path:     path,
		MaxSize:  maxSize,
		Excludes: excludes,
	}
}

func (d *Directory) String() string {
	return "directory:" + d.Path
}

func (d *Directory) Chunks(
	ctx context.Context, out chan<- types.Chunk,
) error {
	return filepath.WalkDir(
		d.Path,
		func(path string, entry fs.DirEntry, err error) error {
			if err != nil {
				return nil //nolint:nilerr
			}

			if ctx.Err() != nil {
				return ctx.Err()
			}

			if entry.IsDir() {
				base := entry.Name()
				if base == ".git" || base == "node_modules" ||
					base == "vendor" || base == "__pycache__" ||
					base == ".venv" || base == "venv" ||
					base == ".svn" || base == ".hg" ||
					base == ".tox" || base == ".mypy_cache" ||
					base == ".pytest_cache" || base == ".ruff_cache" ||
					base == ".next" || base == ".nuxt" ||
					base == ".terraform" || base == ".gradle" ||
					base == "Pods" || base == "coverage" ||
					base == ".nyc_output" || base == ".bundle" ||
					base == "target" || base == ".eggs" {
					return filepath.SkipDir
				}
				return nil
			}

			rel, relErr := filepath.Rel(d.Path, path)
			if relErr != nil {
				rel = path
			}

			if d.isExcluded(rel) {
				return nil
			}

			if isBinaryExt(path) {
				return nil
			}

			info, infoErr := entry.Info()
			if infoErr != nil || info.Size() > d.MaxSize {
				return nil //nolint:nilerr
			}

			return d.emitChunks(ctx, path, rel, out)
		},
	)
}

func (d *Directory) emitChunks(
	ctx context.Context,
	absPath, relPath string,
	out chan<- types.Chunk,
) error {
	f, err := os.Open(absPath) //nolint:gosec
	if err != nil {
		return nil //nolint:nilerr
	}
	defer f.Close() //nolint:errcheck

	var buf strings.Builder
	scanner := bufio.NewScanner(f)
	scanner.Buffer(make([]byte, 0, 512*1024), 512*1024)

	lineNum := 0
	chunkStart := 1
	linesInChunk := 0

	for scanner.Scan() {
		if ctx.Err() != nil {
			return ctx.Err()
		}

		lineNum++
		linesInChunk++

		if buf.Len() > 0 {
			buf.WriteByte('\n')
		}
		buf.WriteString(scanner.Text())

		if linesInChunk >= 50 {
			select {
			case <-ctx.Done():
				return ctx.Err()
			case out <- types.Chunk{
				Content:   buf.String(),
				FilePath:  relPath,
				LineStart: chunkStart,
			}:
			}
			buf.Reset()
			chunkStart = lineNum + 1
			linesInChunk = 0
		}
	}

	if buf.Len() > 0 {
		select {
		case <-ctx.Done():
			return ctx.Err()
		case out <- types.Chunk{
			Content:   buf.String(),
			FilePath:  relPath,
			LineStart: chunkStart,
		}:
		}
	}

	return nil
}

func (d *Directory) isExcluded(relPath string) bool {
	for _, pattern := range d.Excludes {
		if matched, _ := filepath.Match( //nolint:errcheck
			pattern, filepath.Base(relPath),
		); matched {
			return true
		}
		if strings.Contains(relPath, pattern) {
			return true
		}
	}
	return false
}

var binaryExts = map[string]bool{
	".png": true, ".jpg": true, ".jpeg": true,
	".gif": true, ".ico": true, ".svg": true,
	".bmp": true, ".tiff": true, ".tif": true,
	".webp": true, ".avif": true, ".heic": true,
	".heif": true, ".psd": true, ".raw": true,
	".cr2":  true,
	".woff": true, ".woff2": true, ".ttf": true,
	".eot": true, ".otf": true,
	".mp3": true, ".mp4": true, ".avi": true,
	".mkv": true, ".mov": true, ".wmv": true,
	".flv": true, ".webm": true, ".flac": true,
	".wav": true, ".ogg": true, ".aac": true,
	".m4a": true,
	".zip": true, ".tar": true, ".gz": true,
	".rar": true, ".7z": true, ".bz2": true,
	".xz": true, ".zst": true, ".lz4": true,
	".lzma": true, ".cab": true,
	".deb": true, ".rpm": true, ".msi": true,
	".dmg": true, ".iso": true, ".img": true,
	".snap": true,
	".pdf":  true, ".exe": true, ".dll": true,
	".so": true, ".dylib": true, ".bin": true,
	".o": true, ".a": true, ".class": true,
	".jar": true, ".war": true, ".ear": true,
	".pyc": true, ".pyo": true, ".wasm": true,
	".db": true, ".sqlite": true, ".sqlite3": true,
	".dat": true, ".npy": true, ".npz": true,
	".h5": true, ".hdf5": true, ".parquet": true,
	".tfrecord": true, ".pkl": true, ".pickle": true,
	".pt": true, ".onnx": true, ".safetensors": true,
}

func isBinaryExt(path string) bool {
	return binaryExts[strings.ToLower(filepath.Ext(path))]
}