diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 00000000..43fe72ae --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,108 @@ +FROM kalilinux/kali-rolling + +# Set environment variable to non-interactive +ENV DEBIAN_FRONTEND=noninteractive + +# [Optional] Uncomment this section to install additional OS packages. +RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ + && apt-get -y install --no-install-recommends \ + net-tools python3 python3-pip python3-venv \ + curl gnupg nmap iputils-ping ssh git \ + graphviz pkg-config libhdf5-dev \ + build-essential python3-dev \ + asciinema dnsutils \ + apt-transport-https ca-certificates \ + wget dirb gobuster whatweb gfortran \ + cmake libopenblas-dev sshpass seclists \ + golang-go + +# # Install Metasploit - path 1 +# RUN curl https://raw.githubusercontent.com/rapid7/metasploit-omnibus/master/config/templates/metasploit-framework-wrappers/msfupdate.erb > /tmp/msfinstall \ +# && chmod 755 /tmp/msfinstall \ +# && /tmp/msfinstall + +# # Install Metasploit - path 2 +# RUN curl https://apt.metasploit.com/metasploit-framework.gpg.key | apt-key add - +# RUN echo "deb https://apt.metasploit.com/ lucid main" | tee /etc/apt/sources.list.d/metasploit-framework.list +# RUN apt-get update && apt-get install metasploit-framework + +# Install Metasploit - path 3 +RUN curl https://raw.githubusercontent.com/rapid7/metasploit-omnibus/master/config/templates/metasploit-framework-wrappers/msfupdate.erb > msfinstall && chmod 755 msfinstall && ./msfinstall + +# Update Kali Linux repositories +RUN apt-get update \ + && apt-get upgrade -y + +# # Create a virtual environment +# ENV VIRTUAL_ENV=/opt/venv +# RUN python3 -m venv $VIRTUAL_ENV +# ENV PATH="$VIRTUAL_ENV/bin:$PATH" + +# # [Optional] If your pip requirements rarely change, uncomment this section to add them to the image. +# # alternatively, this also helps avoid having to pull requirements from the internet every single time +# COPY requirements.txt /tmp/pip-tmp/ +# RUN pip3 --disable-pip-version-check --no-cache-dir install -r /tmp/pip-tmp/requirements.txt \ +# && rm -rf /tmp/pip-tmp + +# FIX for: +# +# Python packages system-wide, try apt install +# python3-xyz, where xyz is the package you are trying to +# install. +# +# see https://github.com/raspiblitz/raspiblitz/issues/4170 +RUN mkdir -p /root/.pip && touch /root/.pip/pip.conf +RUN echo "[global]" > /root/.pip/pip.conf && \ + echo "break-system-packages = true" >> /root/.pip/pip.conf + +RUN pip3 install --upgrade setuptools +RUN pip3 install pandas \ + opentelemetry-sdk opentelemetry-exporter-otlp \ + mem0ai PyPDF2 sentence_transformers tf-keras \ + pytest-repeat parameterized pytest-rerunfailures \ + pytest-clarity tiktoken blobfile pytest-timeout \ + invoke fabric + +# Install rust compiler - dependency of outlines-core +RUN apt-get update && \ + apt-get install -y curl && \ + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y && \ + . $HOME/.cargo/env && \ + rustup default stable && \ + rustup component add rustfmt clippy && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +ENV PATH="/root/.cargo/bin:${PATH}" +RUN wget https://raw.githubusercontent.com/adfoster-r7/metasploit-info/master/info/module_metadata.json -O /tmp/module_metadata.json + +# NOTE: align dev env in Linux +# work +# RUN echo "192.168.2.1 host.docker.internal" >> /etc/hosts +# Update and upgrade packages +# Preconfigure console-setup and install kali-linux-headless fixers +RUN apt-get update && \ + echo "console-setup console-setup/variant select Latin1 and Latin5 - western Europe and Turkic languages" | debconf-set-selections && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends kali-linux-headless + +# Update pip again -> python3 -m pip install --upgrade pip causes crashes +# Install spacy deps - en_core_web_sm +# RUN python3 -m spacy download en_core_web_sm + +# Install docker inside of dev env +RUN apt-get update && \ + apt-get install -y apt-transport-https ca-certificates curl gnupg && \ + curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg && \ + echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian bullseye stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null && \ + apt-get update && \ + apt-get install -y docker-ce docker-ce-cli containerd.io cloc + +# # Activate virtual environment by default +# RUN echo "source $VIRTUAL_ENV/bin/activate" >> ~/.bashrc + +RUN cargo install --git https://github.com/asciinema/agg + +# Remove system's python3 libs to avoid conflicts +RUN apt-get remove -y python3-jsonschema && \ + apt-get remove -y python3-numpy && \ + apt-get autoremove -y diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 00000000..22e1192e --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,86 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at: +// https://github.com/microsoft/vscode-dev-containers/tree/v0.241.1/containers/python-3 +{ + "name": "cai_devenv", + + // "build": { + // "dockerfile": "Dockerfile", + // "context": "..", + // "args": { + // // Update 'VARIANT' to pick a Python version: 3, 3.10, 3.9, 3.8, 3.7, 3.6 + // // Append -bullseye or -buster to pin to an OS version. + // // Use -bullseye variants on local on arm64/Apple Silicon. + // "VARIANT": "3.10-bullseye", + // // Options + // "NODE_VERSION": "lts/*" + // } + // }, + + "dockerComposeFile": ["./docker-compose.yml"], + "service": "devenv", + // "shutdownAction": "none", // don't shut down container when vscode is closed + "workspaceFolder": "/workspace", + + // Configure tool-specific properties. + "customizations": { + // Configure properties specific to VS Code. + "vscode": { + // Set *default* container specific settings.json values on container create. + "settings": { + "python.defaultInterpreterPath": "/usr/local/bin/python", + "python.linting.enabled": true, + "python.linting.pylintEnabled": true, + "python.formatting.autopep8Path": "/usr/local/py-utils/bin/autopep8", + "python.formatting.blackPath": "/usr/local/py-utils/bin/black", + "python.formatting.yapfPath": "/usr/local/py-utils/bin/yapf", + "python.linting.banditPath": "/usr/local/py-utils/bin/bandit", + "python.linting.flake8Path": "/usr/local/py-utils/bin/flake8", + "python.linting.mypyPath": "/usr/local/py-utils/bin/mypy", + "python.linting.pycodestylePath": "/usr/local/py-utils/bin/pycodestyle", + "python.linting.pydocstylePath": "/usr/local/py-utils/bin/pydocstyle", + "python.linting.pylintPath": "/usr/local/py-utils/bin/pylint" + }, + + // Add the IDs of extensions you want installed when the container is created. + "extensions": [ + "ms-python.python", + "ms-toolsai.jupyter-renderers", + "ms-toolsai.jupyter", + "ms-python.vscode-pylance", + "ms-azuretools.vscode-docker", + "ms-python.debugpy", + "ms-python.black-formatter", + "MS-vsliveshare.vsliveshare" + ] + } + }, + + // Use 'forwardPorts' to make a list of ports inside the container available locally. + // "forwardPorts": [], + + // Use 'preCreateCommand' to run commands before the container is created. + "preCreateCommand": "docker network prune -f && docker container prune -f", + + // Use 'postCreateCommand' to run commands after the container is created. + // "postCreateCommand": "pip3 install --user -r requirements.txt", // doing it instead in + // the dockerfile to cache + // requirements in container + // Install cai via "pip3 install -e ." + // Extensions, NOTE: only one postCreateCommand is allowed, so we need to install all extensions here + "postCreateCommand": "pip3 install -e /workspace/", + + // MSF setup, and RAG setup + "postStartCommand": [ + "nohup", "msfrpcd", "-P", "cai", "&", + "&&", + "python3", "cai/ins/rag/agent_helper.py" + ], + // // Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. + // "remoteUser": "vscode" + + "runArgs": [ + "--privileged", + "-e", "DISPLAY=host.docker.internal:0", + "-v", "/tmp/.X11-unix:/tmp/.X11-unix" + ] +} diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml new file mode 100644 index 00000000..92b14fb9 --- /dev/null +++ b/.devcontainer/docker-compose.yml @@ -0,0 +1,78 @@ +version: '3' + +################# +# SERVICES +################# +services: + # Developer environment + devenv: + build: + context: .. + dockerfile: .devcontainer/Dockerfile + # platform: linux/amd64 # TODO: uncomment this when we have a x86_64 machine + volumes: + # Mount the root folder that contains .git + - ..:/workspace:cached + # Mount Docker socket to enable Docker-in-Docker + - /var/run/docker.sock:/var/run/docker.sock + command: /bin/sh -c "while sleep 10; do :; done" + networks: + cainet: + ipv4_address: 192.168.2.5 + cap_add: + - NET_ADMIN + - SYS_ADMIN + privileged: true + + # # Vulnhub vulnerable machine (Hackable II) SUGGESTION: Comment this and use PentestPerf + # hackableii: + # image: vmayoral/vulnhub:hackableii + # command: | + # /bin/bash -c " + # rm -rf /var/lock && \ + # mkdir -p /var/lock && \ + # chmod 755 /var/lock && \ + # /etc/init.d/apache2 start && \ + # /etc/init.d/ssh start && \ + # /etc/init.d/runproftpd.sh && \ + # /etc/init.d/php7.0-fpm start && \ + # while sleep 1; do :; done + # " + # networks: + # cainet: + # ipv4_address: 192.168.2.11 + # mac_address: 08:00:27:85:55:86 + + # # Vulnhub vulnerable machine (Bob) SUGGESTION: Comment this and use PentestPerf + # bob: + # image: vmayoral/vulnhub:bob + # command: | + # /bin/bash -c "rm -r /var/lock; mkdir -p /var/lock; chmod 755 /var/lock; /etc/init.d/apache2 start; /etc/init.d/ssh start; while sleep 10; do :; done" + # ports: # map port in the container to the host systems + # - "8080:80" + # networks: + # cainet: + # ipv4_address: 192.168.2.12 + # mac_address: 08:00:27:cb:07:d4 + + # Qdrant vector database + qdrant: + image: qdrant/qdrant + ports: + - "6333:6333" + - "6334:6334" + volumes: + - ./qdrant_storage:/qdrant/storage:z + networks: + cainet: + ipv4_address: 192.168.2.13 + +################# +# NETWORKS +################# +networks: + cainet: + ipam: + driver: default + config: + - subnet: 192.168.2.0/24 diff --git a/.devcontainer/qdrant_storage/aliases/data.json b/.devcontainer/qdrant_storage/aliases/data.json new file mode 100644 index 00000000..9e26dfee --- /dev/null +++ b/.devcontainer/qdrant_storage/aliases/data.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/.devcontainer/qdrant_storage/raft_state.json b/.devcontainer/qdrant_storage/raft_state.json index 23b070d6..b7bcb749 100644 --- a/.devcontainer/qdrant_storage/raft_state.json +++ b/.devcontainer/qdrant_storage/raft_state.json @@ -1 +1 @@ -{"state":{"hard_state":{"term":0,"vote":0,"commit":0},"conf_state":{"voters":[6120741261401965],"learners":[],"voters_outgoing":[],"learners_next":[],"auto_leave":false}},"latest_snapshot_meta":{"term":0,"index":0},"apply_progress_queue":null,"first_voter":6120741261401965,"peer_address_by_id":{},"peer_metadata_by_id":{},"this_peer_id":6120741261401965} +{"state":{"hard_state":{"term":0,"vote":0,"commit":0},"conf_state":{"voters":[6120741261401965],"learners":[],"voters_outgoing":[],"learners_next":[],"auto_leave":false}},"latest_snapshot_meta":{"term":0,"index":0},"apply_progress_queue":null,"first_voter":6120741261401965,"peer_address_by_id":{},"peer_metadata_by_id":{},"this_peer_id":6120741261401965} \ No newline at end of file diff --git a/.devcontainer/requirements.txt b/.devcontainer/requirements.txt new file mode 100644 index 00000000..2409daf2 --- /dev/null +++ b/.devcontainer/requirements.txt @@ -0,0 +1,6 @@ +pymetasploit3 +networkx==2.5 +requests +wasabi +xmltodict +pydot==1.4.2 diff --git a/.gitignore b/.gitignore index 5101cdeb..f23dc111 100644 --- a/.gitignore +++ b/.gitignore @@ -30,7 +30,7 @@ share/python-wheels/ .installed.cfg *.egg MANIFEST - +qdrant_storage/ # PyInstaller *.manifest *.spec