Updated scripts

sections/__init__.py

@@ -0,0 +1,94 @@
+"""This handbook has 7 sections covering common tasks for developing
+Python scripts for use in DFIR. Each section contains short,
+portable code blocks that can drop into a new script with minimal
+tweaking. This way, you can quickly build out your custom script
+without needing to re-invent the wheel each time.
+
+This handbook is not intended to be read in order - if anything
+this outline is the main launching point to find the correct page
+containing the code block you wish to reference.
+
+    **Tip**
+
+    On each of the subpages containing descriptions of the code, you
+    can click the ``[source]`` button to the right of the class or
+    function definition and see the source code with the
+    documentation in line.
+
+Section 1 - Essential Script Elements
+-------------------------------------
+
+This chapter covers code blocks that are useful across scripts
+and are not DFIR specific, but solid practices to integrate into
+projects to allow for uniformity.
+
+* Argparse
+    - Command line parameter handling
+* Logging
+    - Writing status and error messages to the console and
+      log file
+* CSV Generation
+    - For better or worse, CSV reports are very common in DFIR
+      and this code block covers several methods for
+      generating a CSV
+* Recursive File Exploration
+    - Quick example of code to explore directories and access
+      nested files.
+* Parallel Processing
+    - Simple implementation of multithreading and multiprocessing
+
+Section 2 - Registry Hives with YARP
+------------------------------------
+
+* Using yarp to open a single hive
+    - Opening a hive and confirming it's the one you want to view
+* Read key information/metadata
+    - USB Devices
+* Read value information/metadata
+    - USB Devices
+* YARP hive file + transaction logs/other registry fragments
+
+Section 3 - Event Logs
+----------------------
+
+* Using python-evtx
+    - Opening evtx files
+* Counts/Metadata about EVTX container
+* Parsing Logins (with types, levels, privs)
+* Parsing Logouts (durations)
+* Parsing Powershell decoding
+
+Section 4 - Text logs
+---------------------
+
+* Handling IIS Logs
+* Handling Syslog
+* Adding in GeoIP
+
+Section 5 - API calls & JSON data
+---------------------------------
+
+* VirusTotal
+* HybridAnalysis
+* Manipulating JSON
+
+Section 6 - SQLite & macOS/mobile/browsers
+------------------------------------------
+
+* macOS Activity
+    - KnowledgeC
+* Andriod SMS
+* Google Chome History DB
+
+Section 7 - Opening forensic images
+--------------------------------------
+
+* LibEWF
+    - Expose an E01 as a raw image
+* PyTSK
+    - Read data from a raw image (MBR)
+    - Read data from a file (hashing)
+    - Iterate through folders (file listing)
+    - Perform targetted reads (file sigs)
+
+"""

sections/section_01/__init__.py

@@ -0,0 +1,7 @@
+"""Essential Script Snippets
+
+In this section, we will cover developing essential code snippets.
+These code blocks are commonly used in DFIR scripting to allow
+for users to provide parameters to the tool, logging of progress
+and errors, and generation of reports.
+"""

sections/section_01/argparse_example.py

@@ -1,3 +1,28 @@
+"""Example for setting up arguments for your command line utility.
+
+Example Usage:
+
+    ``$ python argparse.py``
+
+References:
+
+* https://docs.python.org/3/library/argparse.html
+* https://docs.python.org/3/library/os.html
+* https://docs.python.org/3/library/pathlib.html
+
+Argparse configuration
+======================
+
+This function shows an example of creating an argparse instance
+with required and optional parameters. Further, it demonstrates
+how to set default values and boolean arguments. the ``argparse``
+module has many more features documented at
+https://docs.python.org/3/library/argparse.html
+
+.. literalinclude:: ../sections/section_01/argparse_example.py
+    :pyobject: setup_argparse
+
+"""
 import argparse
 import os
 from pathlib import PurePath
@@ -36,28 +61,48 @@ __docs__ = [
     'https://docs.python.org/3/library/pathlib.html'
 ]
 
-# Only run if called directly (not imported)
-if __name__ == '__main__':
+def setup_argparse():
+    # Setup a parser instance with common fields including a
+    # description and epilog. The `formatter_class` instructs
+    # argparse to show default values set for parameters.
     parser = argparse.ArgumentParser(
         description='Sample Argparse',
         formatter_class=argparse.ArgumentDefaultsHelpFormatter,
         epilog=f"Built by {__author__}, v.{__date__}"
     )
+
+    # The simplest form of adding an argument, the name of the
+    # parameter and a description of its form.
     parser.add_argument('INPUT_FILE', help="Input file to parse")
     parser.add_argument('OUTPUT_FOLDER',
         help="Folder to store output")
+
+    # An optional argument with multiple methods of specifying
+    # the parameter. Includes a default value
     parser.add_argument('-l', '--log', help="Path to log file",
         default=os.path.abspath(os.path.join(
             PurePath(__file__).parent,
             PurePath(__file__).name.rsplit('.', 1)[0] + '.log'))
     )
+
+    # An optional argument which does not accept a value, instead
+    # just modifies functionality.
     parser.add_argument('-v', '--verbose', action='store_true',
         help='Include debug log messages')
 
+    # Once we've specified our arguments we can parse them for
+    # reference
     args = parser.parse_args()
 
+    # Returning our parsed arguments for further use.
+    return args
+
+# Only run if called directly (not imported)
+if __name__ == '__main__':
+    args = setup_argparse()
+
     # Show arguments
     print(f'Input file: {args.INPUT_FILE}')
     print(f'Output folder: {args.OUTPUT_FOLDER}')
     print(f'Log file: {args.log}')
-    print(f'Be verbose?: {args.verbose}')
+    print(f'Be verbose?: {args.verbose}')

sections/section_01/csv_example.py

@@ -1,5 +1,72 @@
+"""Example for writing datasets into CSV files.
+
+Demonstrates source datasets comprised of lists of dictionaries
+and lists of lists as separate functions. Example data is
+provided in line and will generate two identical CSVs as output.
+
+Example Usage:
+
+    ``$ python csv_example.py``
+
+References:
+
+* https://docs.python.org/3/library/csv.html
+* https://docs.python.org/3/library/os.html
+
+
+List of dictionaries to CSV
+===========================
+
+Example ``data`` variable:
+
+::
+
+    [
+        {'name': 'apple', 'quantity': 10, 'location': 'VT'},
+        {'name': 'orange', 'quantity': 5, 'location': 'FL'}
+    ]
+
+This first function shows an example of writing a list containing
+multiple dictionaries to a CSV file. You can optionally provide
+an ordered list of headers to filter what rows to show, or let the
+function use the keys of the first dictionary in the list to
+generate the header information. The latter option may produce
+a new order each iteration and is not prefered if you can
+determine the headers in advance.
+
+.. literalinclude:: ../sections/section_01/csv_example.py
+    :pyobject: write_csv_dicts
+
+List of ordered lists to CSV
+============================
+
+Example ``data`` variable:
+
+::
+
+    [
+        ['name', 'quantity', 'location'],
+        ['apple', 10, 'VT'],
+        ['orange', 5, 'FL']
+    ]
+
+This function shows an example of writing a list containing
+multiple lists to a CSV file. You can optionally provide
+an ordered list of headers, or let the function use the values
+of the first element in the list to generate the header
+information. Unlike the dictionary option, you cannot filter
+column data by adjusting the provided headers, you must write all
+columns to the CSV.
+
+.. literalinclude:: ../sections/section_01/csv_example.py
+    :pyobject: write_csv_lists
+
+
+Docstring References
+====================
+"""
+
 import csv
-import os
 
 """
 Copyright 2019 Chapin Bryce
@@ -89,7 +156,7 @@ sample_dict_data = [
 
 write_csv_dicts('dict_test.csv', sample_dict_data)
 
-headers = ['id', 'city', 'state', 'country']
+header_row = ['id', 'city', 'state', 'country']
 sample_list_data = [
     ['0', 'Boston', 'MA', 'USA'],
     ['1', 'New York', 'NY', 'USA'],
@@ -97,4 +164,4 @@ sample_list_data = [
 ]
 
 write_csv_lists('list_test.csv', sample_list_data,
-                headers=headers)
+                headers=header_row)

sections/section_01/logging_example.py

@@ -0,0 +1,113 @@
+"""Example for writing logging information to the console and a
+log file.
+
+Example Usage:
+
+    ``$ python logging_example.py``
+
+References:
+
+* https://docs.python.org/3/library/logging.html
+* https://docs.python.org/3/library/os.html
+
+Logging configuration
+=====================
+
+This function shows an example of creating a logging instance that
+writes messages to both STDERR and a file, allowing your script
+to write content to STDOUT uninterrupted. Additionally, you can
+set different logging levels for the two handlers - generally you
+keep debugging information in the log file while writing more
+critical messages to the console in STDERR.
+
+.. literalinclude:: ../sections/section_01/logging_example.py
+    :pyobject: setup_logging
+
+Docstring References
+====================
+"""
+import logging
+import sys
+
+"""
+Copyright 2019 Chapin Bryce
+
+Permission is hereby granted, free of charge, to any person
+obtaining a copy of this software and associated documentation
+files (the "Software"), to deal in the Software without
+restriction, including without limitation the rights to use, copy,
+modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+"""
+
+__author__ = 'Chapin Bryce'
+__date__ = 20190527
+__license__ = 'MIT Copyright 2019 Chapin Bryce'
+__desc__ = '''Sample script to display and write logging
+    messages.'''
+__docs__ = [
+    'https://docs.python.org/3/library/logging.html',
+    'https://docs.python.org/3/library/os.html'
+]
+
+logger = None
+
+def setup_logging():
+    """Function to setup logging configuration and test it."""
+    # Allow us to modify the `logger` variable within a function
+    global logger
+
+    # Set logger object, uses module's name
+    logger = logging.getLogger(name=__name__)
+
+    # Set default logger level to DEBUG. You can change this later
+    logger.setLevel(logging.DEBUG)
+
+    # Logging formatter. Best to keep consistent for most usecases
+    log_format = logging.Formatter(
+        '%(asctime)s %(filename)s %(levelname)s %(module)s '
+        '%(funcName)s %(lineno)d %(message)s')
+
+    # Setup STDERR logging, allowing you uninterrupted
+    # STDOUT redirection
+    stderr_handle = logging.StreamHandler(stream=sys.stderr)
+    stderr_handle.setLevel(logging.INFO)
+    stderr_handle.setFormatter(log_format)
+
+    # Setup file logging
+    file_handle = logging.FileHandler('sample.log', 'a')
+    file_handle.setLevel(logging.DEBUG)
+    file_handle.setFormatter(log_format)
+
+    # Add handles
+    logger.addHandler(stderr_handle)
+    logger.addHandler(file_handle)
+
+    # Sample log messages
+    logger.debug("This is a debug message")
+    logger.info("This is an info message")
+    logger.warning("This is a warning message")
+    logger.error("This is a error message")
+    logger.critical("This is a critical message")
+
+    def sample_function():
+        """Sample function to demonstrate logging formatting."""
+        logger.info("Called from a function")
+
+    sample_function()
+
+if __name__ == "__main__":
+    setup_logging()

sections/section_01/recursion_example.py

@@ -0,0 +1,110 @@
+"""File recursion example.
+
+Demonstration of iterating through a directory to interact with
+files.
+
+Example Usage:
+
+    ``$ python recursion_example.py``
+
+References:
+
+* https://docs.python.org/3/library/os.html
+
+List a directory
+================
+
+This function shows an example of displaying all files and
+folders within a single directory. From here you can further
+interact with individual files and folders or iterate recursively
+by calling the function on identified subdirectories.
+
+.. literalinclude:: ../sections/section_01/recursion_example.py
+    :pyobject: list_directory
+
+List a directory recursively
+============================
+
+This function shows an example of displaying all files and
+folders within a all directories. You don't need to worry about
+additional function calls as the ``os.walk()`` method handles
+the recursion on subdirectories and your logic can focus on
+handling the processing of files. This sample shows a method of
+counting the number of files, subdirectories, and files ending in
+".py" as an example.
+
+.. literalinclude:: ../sections/section_01/recursion_example.py
+    :pyobject: iterate_files
+
+"""
+import os
+
+"""
+Copyright 2019 Chapin Bryce
+
+Permission is hereby granted, free of charge, to any person
+obtaining a copy of this software and associated documentation
+files (the "Software"), to deal in the Software without
+restriction, including without limitation the rights to use, copy,
+modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+"""
+
+__author__ = 'Chapin Bryce'
+__date__ = 20190527
+__license__ = 'MIT Copyright 2019 Chapin Bryce'
+__desc__ = '''Sample script to iterate over a folder of files.'''
+__docs__ = [
+    'https://docs.python.org/3/library/os.html'
+]
+
+def list_directory(path):
+    """List all file and folder entries in `path`."""
+    print(f"Files and folders in '{os.path.abspath(path)}':")
+    # Quick and easy method for listing items within a single
+    # folder.
+    for entry in os.listdir(path):
+        # Print all entry names
+        print(f"\t{entry}")
+
+def iterate_files(path):
+    """Iterate over the `path` recursively."""
+    num_files = 0
+    num_py_files = 0
+    num_folders = 0
+    # Though `os.walk()` exposes a list of directories in the
+    # current `root`, it is rarely used since we are generally
+    # interested in the files found within the subdirectories.
+    # For this reason, it is common to see `dirs` named `_`.
+    # DO NOT NAME `dirs` as `dir` since `dir` is a reserved word!
+    for root, dirs, files in os.walk(os.path.abspath(path)):
+        # Both `dirs` and `files` are lists containing all entries
+        # at the current `root`. This means we can quickly count
+        # the number of files and folders by taking the `len()`.
+        num_folders += len(dirs)
+        num_files += len(files)
+        for fentry in files:
+            # To effectively reference a file, you should include
+            # the below line which creates a full path reference
+            # to the specific file, regardless of how nested it is
+            file_entry = os.path.join(root, file_entry)
+            # We can then hand `file_entry` off to other functions.
+            if file_entry.endswith('py'):
+                num_py_files += 1
+
+if __name__ == "__main__":
+    list_directory('.')
+    iterate_files('../../')

sections/section_01_essentials/logging_example.py

@@ -1,73 +0,0 @@
-import logging
-import sys
-
-"""
-Copyright 2019 Chapin Bryce
-
-Permission is hereby granted, free of charge, to any person
-obtaining a copy of this software and associated documentation
-files (the "Software"), to deal in the Software without
-restriction, including without limitation the rights to use, copy,
-modify, merge, publish, distribute, sublicense, and/or sell copies
-of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
-OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
-HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
-WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE.
-"""
-
-__author__ = 'Chapin Bryce'
-__date__ = 20190527
-__license__ = 'MIT Copyright 2019 Chapin Bryce'
-__desc__ = '''Sample script to display and write logging
-    messages.'''
-__docs__ = [
-    'https://docs.python.org/3/library/logging.html',
-    'https://docs.python.org/3/library/os.html'
-]
-
-# Set logger object, uses module's name
-logger = logging.getLogger(name=__name__)
-
-# Set default logger level to DEBUG. You can change this later
-logger.setLevel(logging.DEBUG)
-
-# Logging formatter. Best to keep consistent for most usecases
-log_format = logging.Formatter(
-    '%(asctime)s %(filename)s %(levelname)s %(module)s '
-    '%(funcName)s %(lineno)d %(message)s')
-
-# Setup STDERR logging
-stderr_handle = logging.StreamHandler(stream=sys.stderr)
-stderr_handle.setLevel(logging.INFO)
-stderr_handle.setFormatter(log_format)
-
-# Setup file loggings
-file_handle = logging.FileHandler('sample.log', 'a')
-file_handle.setLevel(logging.DEBUG)
-file_handle.setFormatter(log_format)
-
-# Add handles
-logger.addHandler(stderr_handle)
-logger.addHandler(file_handle)
-
-# Sample log messages
-logger.debug("This is a debug message")
-logger.info("This is an info message")
-logger.warning("This is a warning message")
-logger.error("This is a error message")
-logger.critical("This is a critical message")
-
-def sample_function():
-    logger.info("Called from a function")
-
-sample_function()