chore: release v2.3.4

* linting

* adding py.typed

* adding pyyaml types

* workaround #1

* updating latest

* updating dev dependencies

* adding isort mypy safety

* chore: prepare release 2.3.3

.coveragerc

@@ -0,0 +1,7 @@
+[report]
+exclude_lines =
+    # Have to re-enable the standard pragma
+    pragma: no cover
+
+    # Don't complain if tests don't hit defensive assertion code:
+    raise NotImplementedError

.gitignore

@@ -69,7 +69,7 @@ instance/
 .scrapy
 
 # Sphinx documentation
-docs/_build/
+docs/source/_build/
 
 # PyBuilder
 target/
@@ -127,3 +127,6 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
+
+# custom
+.DS_Store

.vscode/settings.json

@@ -0,0 +1,8 @@
+{
+    "python.linting.pylintEnabled": false,
+    "python.linting.prospectorEnabled": true,
+    "python.linting.enabled": true,
+    "python.pythonPath": ".venv/bin/python",
+    "restructuredtext.confPath": "${workspaceFolder}/docs/source",
+    "peacock.color": "#307E6A"
+}

CHANGELOG.md

@@ -0,0 +1,24 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+<!-- marker -->
+## [v2.3.4](https://github.com/dtomlinson91/panaetius/commits/v2.3.4) - 2022-01-03
+<small>[Compare with 2.3.3](https://github.com/dtomlinson91/panaetius/compare/2.3.3..v2.3.4)</small>
+
+### Documentation
+
+- Update README.md with script boilerplate ([179007](https://github.com/dtomlinson91/panaetius/commit/1790071741207de13330ba75d7bf090106290d72))
+- Update README.md with script quickstart logging ([e6cfde](https://github.com/dtomlinson91/panaetius/commit/e6cfded87dcfc5d2bf62d36bc7b4dbbdeb94b0b8))
+
+### Features
+
+- Add ability to retrieve keys 3 levels deep ([f5ea19](https://github.com/dtomlinson91/panaetius/commit/f5ea19e7d2f977244594b378c6b7633f02f6048a))
+
+### Testing
+
+- Add tests for f5ea19e ([156af4](https://github.com/dtomlinson91/panaetius/commit/156af4685510bac97a850b83d63f8337635db199))
+
+### Build
+
+- Update dependencies ([9bc89f](https://github.com/dtomlinson91/panaetius/commit/9bc89fd2ce9ddf8dcd6a3ca84ef9b72ee183efd3))

README.md

@@ -0,0 +1,119 @@
+# Panaetius
+
+This package provides:
+
+- Functionality to read user variables from a `config.yml` or environment variables.
+- A convenient default logging formatter printing `json` that can save to disk and rotate.
+- Utility functions.
+
+## Config
+
+### options
+
+#### skip_header_init
+
+If `skip_header_init=True` then the `config_path` will not use the `header_variable` as the
+sub-directory in the `config_path`.
+
+E.g
+
+`CONFIG = panaetius.Config("tembo", "~/tembo/.config", skip_header_init=True)`
+
+Will look in `~/tembo/config/config.yml`.
+
+If `skip_header_init=False` then would look in `~/tembo/config/tembo/config.yml`.
+
+### Module
+
+Convenient to place in a package/sub-package `__init__.py`.
+
+See Tembo for an example: <https://github.com/tembo-pages/tembo-core/blob/main/tembo/cli/__init__.py>
+
+Example snippet to use in a module:
+
+```python
+"""Subpackage that contains the CLI application."""
+
+import os
+from typing import Any
+
+import panaetius
+from panaetius.exceptions import LoggingDirectoryDoesNotExistException
+
+
+if (config_path := os.environ.get("TEMBO_CONFIG")) is not None:
+    CONFIG: Any = panaetius.Config("tembo", config_path, skip_header_init=True)
+else:
+    CONFIG = panaetius.Config(
+        "tembo", "~/tembo/.config", skip_header_init=True
+    )
+
+
+panaetius.set_config(CONFIG, "base_path", "~/tembo")
+panaetius.set_config(CONFIG, "template_path", "~/tembo/.templates")
+panaetius.set_config(CONFIG, "scopes", {})
+panaetius.set_config(CONFIG, "logging.level", "DEBUG")
+panaetius.set_config(CONFIG, "logging.path")
+
+try:
+    logger = panaetius.set_logger(
+        CONFIG, panaetius.SimpleLogger(logging_level=CONFIG.logging_level)
+    )
+except LoggingDirectoryDoesNotExistException:
+    _LOGGING_PATH = CONFIG.logging_path
+    CONFIG.logging_path = ""
+    logger = panaetius.set_logger(
+        CONFIG, panaetius.SimpleLogger(logging_level=CONFIG.logging_level)
+    )
+    logger.warning("Logging directory %s does not exist", _LOGGING_PATH)
+
+```
+
+This means in `./tembo/cli/cli.py` you can
+
+```python
+import tembo.cli
+
+# access the CONFIG instance + variables from the config.yml
+tembo.cli.CONFIG
+```
+
+### Script
+
+Create `./config/config.yml` in the same directory as the script.
+
+In the script initialise a `CONFIG` object:
+
+```python
+import pathlib
+
+import panaetius
+
+CONFIG = panaetius.Config(
+    "teenagers_scraper", str(pathlib.Path(__file__).parents[0] / ".config"), skip_header_init=True
+)
+```
+
+Set variables in the same way as the module above.
+
+#### quickstart logging
+
+```python
+import panaetius
+
+
+def get_logger():
+    logging_dir = pathlib.Path(__file__).parents[0] / "logs"
+    logging_dir.mkdir(parents=True, exist_ok=True)
+
+    CONFIG = panaetius.Config("training_data_into_gcp", skip_header_init=True)
+    panaetius.set_config(CONFIG, "logging.level", "DEBUG")
+    panaetius.set_config(CONFIG, "logging.path", logging_dir)
+    return panaetius.set_logger(CONFIG, panaetius.SimpleLogger(logging_level=CONFIG.logging_level))
+```
+
+## Utility Functions
+
+### Squasher
+
+Squashes a json object or Python dictionary into a single level dictionary.

README.md.old

@@ -1,33 +0,0 @@
-# Author
-
-Daniel Tomlinson (dtomlinson@panaetius.co.uk)
-
-# Requires
-
-`>= python3.7`
-
-# Python requirements
-
-- toml = "^0.10.0"
-- pylite = "^0.1.0"
-
-# Documentation
-
-_soon_
-
-# Installation
-
-_soon_
-
-# Easy Way
-
-## Python
-
-### From pip
-
-### From local wheel
-
-### From source
-
-# Example Usage
-

README.rst

@@ -1,62 +0,0 @@
-Author
-=======
-
-Daniel Tomlinson (dtomlinson@panaetius.co.uk)
-
-Requires
-=========
-
-`>= python3.7`
-
-Python requirements
-====================
-
-- toml = "^0.10.0"
-- pylite = "^0.1.0"
-
-Documentation
-==============
-
-Read the documentation on `read the docs`_.
-
-.. _read the docs: https://panaetius.readthedocs.io/en/latest/introduction.html
-
-Installation
-==============
-
-You can install ``panaetius`` the following ways:
-
-Python
--------
-
-.. Attention:: You should install in a python virtual environment
-
-From pypi using pip
-~~~~~~~~~~~~~~~~~~~~
-
-.. code-block:: bash
-
-    pip install panaetius
-
-From local wheel
-~~~~~~~~~~~~~~~~~
-
-Download the latest verion from the `releases`_ page.
-
-.. _releases: https://github.com/dtomlinson91/panaetius/releases
-
-Install with pip:
-
-.. code-block:: bash
-
-    pip install -U panaetius-1.0.2-py3-none-any.whl
-
-
-From source
-~~~~~~~~~~~~
-
-Clone the repo and install using ``setup.py``:
-
-.. code-block:: bash
-
-    python setup.py

cliff.toml

@@ -0,0 +1,57 @@
+# configuration file for git-cliff (0.1.0)
+
+[changelog]
+# changelog header
+header = """
+# Changelog
+All notable changes to this project will be documented in this file.\n
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).\n
+"""
+# template for the changelog body
+# https://tera.netlify.app/docs/#introduction
+body = """
+{% if version %}\
+    ## [{{ version }}](https://github.com/dtomlinson91/panaetius/commits/{{ version }}) - {{ timestamp | date(format="%Y-%m-%d") }}
+    {% if previous.version %}\
+    <small>[Compare with {{ previous.version }}](https://github.com/dtomlinson91/panaetius/compare/{{ previous.version }}...{{ version }})</small>
+    {% endif %}\
+{% else %}\
+    ## [unreleased]
+{% endif %}\
+{% for group, commits in commits | group_by(attribute="group") %}
+    ### {{ group | upper_first }}
+    {% for commit in commits %}
+        - {{ commit.message | upper_first }} ([{{ commit.id | truncate(length=6, end="") }}](https://github.com/dtomlinson91/panaetius/commit/{{ commit.id }}))\
+    {% endfor %}
+{% endfor %}\n
+"""
+# remove the leading and trailing whitespaces from the template
+trim = true
+# changelog footer
+footer = """
+"""
+
+[git]
+# allow only conventional commits
+# https://www.conventionalcommits.org
+conventional_commits = true
+# regex for parsing and grouping commits
+commit_parsers = [
+    { message = "^feat", group = "Features"},
+    { message = "^fix", group = "Bug Fixes"},
+    { message = "^doc", group = "Documentation"},
+    { message = "^perf", group = "Performance"},
+    { message = "^refactor", group = "Refactor"},
+    { message = "^style", group = "Styling"},
+    { message = "^test", group = "Testing"},
+    { message = "^chore\\(release\\): prepare for", skip = true},
+    { message = "^chore", group = "Miscellaneous Tasks", skip = true},
+    { body = ".*security", group = "Security"},
+]
+# filter out the commits that are not matched by commit parsers
+filter_commits = false
+# glob pattern for matching git tags
+tag_pattern = "v[0-9]*"
+# regex for skipping tags
+skip_tags = "v0.1.0-beta.1"

docs/Makefile

@@ -1,20 +0,0 @@
-# Minimal makefile for Sphinx documentation
-#
-
-# You can set these variables from the command line, and also
-# from the environment for the first two.
-SPHINXOPTS    ?=
-SPHINXBUILD   ?= sphinx-build
-SOURCEDIR     = source
-BUILDDIR      = build
-
-# Put it first so that "make" without argument is like "make help".
-help:
-	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
-
-.PHONY: help Makefile
-
-# Catch-all target: route all unknown targets to Sphinx using the new
-# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
-%: Makefile
-	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/make.bat

@@ -1,35 +0,0 @@
-@ECHO OFF
-
-pushd %~dp0
-
-REM Command file for Sphinx documentation
-
-if "%SPHINXBUILD%" == "" (
-	set SPHINXBUILD=sphinx-build
-)
-set SOURCEDIR=source
-set BUILDDIR=build
-
-if "%1" == "" goto help
-
-%SPHINXBUILD% >NUL 2>NUL
-if errorlevel 9009 (
-	echo.
-	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
-	echo.installed, then set the SPHINXBUILD environment variable to point
-	echo.to the full path of the 'sphinx-build' executable. Alternatively you
-	echo.may add the Sphinx directory to PATH.
-	echo.
-	echo.If you don't have Sphinx installed, grab it from
-	echo.http://sphinx-doc.org/
-	exit /b 1
-)
-
-%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
-goto end
-
-:help
-%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
-
-:end
-popd

docs/source/_static/custom.css

@@ -1,87 +0,0 @@
-@import url("css/theme.css");
-
-.modname {
-    font-size: 0.8em;
-    opacity: 0.4;
-}
-.modname::before {
-    content: '- ';
-}
-.title {
-    font-weight: bold;
-    font-size: 1.2em;
-    background-color: #eee;
-    display: block;
-    padding: 1px 5px;
-    border-left: 2px solid #ddd;
-}
-
-/*colour of the text in the toc*/
-.wy-menu-vertical header, .wy-menu-vertical p.caption{
-    color: #b26d46;;
-}
-
-/*colour of the top left header*/
-.wy-side-nav-search{
-    background-color: #31465a;
-}
-
-/*colours of the class definitions*/
-.rst-content dl:not(.docutils) dt{
-    background: #e2d7d1;
-    color: #0b2852;
-    border-top: solid 3px #31465a;
-}
-
-/*colour of the link in the class defintions*/
-.rst-content .viewcode-link, .rst-content .viewcode-back{
-    color: #4b674a;
-}
-
-/*colour of the function definitions*/
-.rst-content dl:not(.docutils) dl dt{
-    border-left: solid 3px #31465a;
-    background: #e2d7d1;
-    color: #0b2852;
-}
-
-/*colour of the link in the function definitions*/
-.rst-content .viewcode-link, .rst-content .viewcode-back{
-    color: #4b674a;
-}
-
-/*edit the width of the body*/
-.wy-nav-content{
-    max-width: 1200px;
-}
-
-/*code example blocks*/
-.rst-content pre.literal-block, .rst-content div[class^="highlight"] pre, .rst-content .linenodiv pre{
-    background: #b4bec8;
-    color: black;
-    /*border-style: solid;*/
-    /*border-width: thin;*/
-}
-
-/*colour of inline code blocks using ``*/
-.rst-content tt.literal, .rst-content tt.literal, .rst-content code.literal{
-    color: #b26d46;
-}
-
-/* Change code blocks font and bump up font size slightly (normally 12px)*/
-.rst-content pre.literal-block, .rst-content div[class^="highlight"] pre, .rst-content .linenodiv pre {
-    font-family: 'Inconsolata', monospace !important;
-    font-size: 14px !important;
-    white-space: pre-wrap;
-}
-
-/* Change code descriptions and literal blocks (inline code via ``) to match the normal font size being used in the sphinx_RTD_theme text  (normally 14px)*/
-.rst-content dl:not(.docutils) tt.descname,.rst-content dl:not(.docutils) tt.descclassname,.rst-content dl:not(.docutils) tt.descname,.rst-content dl:not(.docutils) code.descname,.rst-content dl:not(.docutils) tt.descclassname,.rst-content dl:not(.docutils) code.descclassname, code.docutils {
-    font-family: 'Inconsolata', monospace !important;
-    font-size: 14px !important;
-}
-
-/*variables text*/
-dl.class > dd > table.docutils.field-list tbody tr.field-odd.field th.field-name::before{
-    content: '(Class Attributes) ';
-}

docs/source/changelog.rst

@@ -1,12 +0,0 @@
-Version history
-================
-
-1.0.2
-------
-
-- Minor fixes and documentation updates.
-
-1.0
---------
-
-- Initial release.

docs/source/conf.py

@@ -1,95 +0,0 @@
-# Configuration file for the Sphinx documentation builder.
-#
-# This file only contains a selection of the most common options. For a full
-# list see the documentation:
-# https://www.sphinx-doc.org/en/master/usage/configuration.html
-
-# -- Path setup --------------------------------------------------------------
-
-# If extensions (or modules to document with autodoc) are in another directory,
-# add these directories to sys.path here. If the directory is relative to the
-# documentation root, use os.path.abspath to make it absolute, like shown here.
-#
-# import os
-# import sys
-# sys.path.insert(0, os.path.abspath('.'))
-import panaetius
-from panaetius.__version__ import __version__ as version
-import sphinx_rtd_theme
-
-# -- Project information -----------------------------------------------------
-
-project = 'panaetius'
-copyright = '2019, Daniel Tomlinson'
-author = 'Daniel Tomlinson'
-
-# The full version, including alpha/beta/rc tags
-release = version
-
-
-# -- General configuration ---------------------------------------------------
-
-# Add any Sphinx extension module names here, as strings. They can be
-# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
-# ones.
-extensions = [
-    'sphinx.ext.autodoc',
-    'sphinx.ext.viewcode',
-    'sphinx.ext.napoleon',
-    'sphinx.ext.todo',
-]
-
-# -- Napoleon Settings -----------------------------------------------------
-napoleon_google_docstring = False
-napoleon_numpy_docstring = True
-napoleon_include_init_with_doc = True
-napoleon_include_private_with_doc = False
-napoleon_include_special_with_doc = False
-napoleon_use_admonition_for_examples = False
-napoleon_use_admonition_for_notes = False
-napoleon_use_admonition_for_references = False
-napoleon_use_ivar = True
-napoleon_use_param = True
-napoleon_use_rtype = True
-napoleon_use_keyword = True
-autodoc_member_order = 'bysource'
-
-# Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
-
-# The master toctree document.
-master_doc = 'index'
-
-# List of patterns, relative to source directory, that match files and
-# directories to ignore when looking for source files.
-# This pattern also affects html_static_path and html_extra_path.
-exclude_patterns = []
-
-
-# -- Options for HTML output -------------------------------------------------
-
-# The theme to use for HTML and HTML Help pages.  See the documentation for
-# a list of builtin themes.
-#
-html_theme = "sphinx_rtd_theme"
-
-# Add any paths that contain custom static files (such as style sheets) here,
-# relative to this directory. They are copied after the builtin static files,
-# so a file named "default.css" will overwrite the builtin "default.css".
-
-html_theme = "sphinx_rtd_theme"
-html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
-html_static_path = ['_static']
-html_context = {'css_files': ['_static/custom.css']}
-html_theme_options = {
-    'collapse_navigation': True,
-    'display_version': True,
-    'prev_next_buttons_location': 'both',
-    #'navigation_depth': 3,
-}
-
-# The name of the Pygments (syntax highlighting) style to use.
-pygments_style = 'sphinx'
-
-# Enable todo
-todo_include_todos = True

docs/source/configuration.rst

@@ -1,23 +0,0 @@
-Configuration
-=============
-
-panaetius is fairly easy to configure. There are just a couple of options to be aware of.
-
-header.py
-----------
-
-You should set a ``__header__.py`` next to your script or module.
-
-This ``__header__.py`` should contain a ``__header__`` variable that sets the name of your project/script.
-
-E.g a ``__header__.py`` for the module ``plex_posters`` would look like:
-
-.. code-block:: python
-
-    __header__ = 'plex_posters' 
-
-If you are writing a script, simply place this ``__header__.py`` along side your script. Panaetius will pick this up when the script is ran.
-
-If you are writing a module, you can either place the ``__header__.py`` alongside the script that uses your module. If this is not possible, panaetius will set the default ``__header__`` variable to the name of the virtualenv that the script is activated from.
-
-If neither of the above aren't possible (say your script is running in a lambda on AWS), then ``__header__`` will be set to the default of ``panaetius``.

docs/source/global.rst

@@ -1,5 +0,0 @@
-.. role:: modname
-   :class: modname
-
-.. role:: title
-   :class: title

docs/source/index.rst

@@ -1,3 +0,0 @@
-Table of Contents
-=================
-.. include:: toc.rst

docs/source/introduction.rst

@@ -1,154 +0,0 @@
-panaetius
-==========
-
-.. image:: https://img.shields.io/readthedocs/panaetius?style=for-the-badge   :target: https://panaetius.readthedocs.io/en/latest/?badge=latest
-   :alt: Documentation Status
-
-.. image:: https://img.shields.io/github/v/tag/dtomlinson91/panaetius?style=for-the-badge   :alt: GitHub tag (latest by date)
-
-.. image:: https://img.shields.io/github/commit-activity/m/dtomlinson91/panaetius?style=for-the-badge   :alt: GitHub commit activity
-
-.. image:: https://img.shields.io/github/issues/dtomlinson91/panaetius?style=for-the-badge   :alt: GitHub issues
-
-.. image:: https://img.shields.io/github/license/dtomlinson91/panaetius?style=for-the-badge   :alt: GitHubtbc
-
-Python module to gracefully handle a .config file/environment variables for scripts, with built in masking for sensitive options. Provides a Splunk friendly formatted logger instance.
-
-Usage
-------
-
-Setting a config file
-~~~~~~~~~~~~~~~~~~~~~~
-
-The main functionality of ``panaetius`` is using a config file to store variables.
-
-Your ``config.toml`` can be created and found in ~/.config/__header__/config.toml where __header__ is equal to the variable configured/set. `See how to configure`_ this variable in the configuration section of panaetius.
-
-.. _See how to configure: https://panaetius.readthedocs.io/en/latest/configuration.html#header-py
-
-Setting values in a config.toml/environment variables
-#######################################################
-
-A ``config.toml`` can be created in the default folder for the module. In this example this would be found in ``~/.config/example_module/config.toml``. 
-
-An example ``config.toml`` could look like:
-
-.. code-block:: toml
-
-    [example_module]
-    test = "a6cbf36649b029f3618a0cc1"
-
-    [example_module.logging]
-    path = "~/.config/example_module"
-    level = "DEBUG"
-
-    [example_module.foo]
-    bar = "6b3b96815218960ceaf7cceb"
-
-These are equivalent to the environment variables:
-
-.. code-block:: bash
-
-    EXAMPLE_MODULE_TEST
-    EXAMPLE_MODULE_LOGGING_PATH
-    EXAMPLE_MODULE_LOGGING_LEVEL
-    EXAMPLE_MODULE_FOO_BAR
-
-
-.. Attention:: 
-    Environment variables take precedent over the ``config.toml``. If both are set then the environment variable will be used.
-
-You can overwrite the ``config.toml`` location by setting the environment variable:
-
-.. code-block:: bash
-
-    DEFAULT_CONFIG_PATH = "~/path/to/config"
-
-
-Setting values in your code
-############################
-
-Values in a ``config.toml`` or from an environment variable need to be set in your work in order for you to use them. You can do this easily by
-
-- importing panaetius.
-- using the :func:`~panaetius.library.set_config` function.
-  
-E.g your script could contain:
-
-.. code-block:: python
-
-    import panaetius
-    panaetius.set_config(panaetius.CONFIG, 'logging.path')
-
-.. Note::
-
-    The ``key`` attribute in :func:`~panaetius.library.set_config` is specified as a string, with the hirearchy in the config file split with a ``.``
-
-.. Important::
-
-    The default value for a variable defined using :func:`~panaetius.library.set_config` is ``None``. See the documentation of this function to see all the options available.
-
-
-Accessing values
-#################
-
-You can then access the result of this variable later in your code:
-
-.. code-block:: python
-
-    panaetius.CONFIG.logging_path
-
-
-Logging
-~~~~~~~~
-
-In order to save to disk, you need to specify a path for the log file in the config file/environment variable. There is no need to register this with :func:`~panaetius.library.set_config` as ``panaetius`` will do this automatically.
-
-There are other options available for you to configure a logger. These are (including the default values which can be overwritten):
-
-.. code-block:: toml
-
-    [example_module.logging]
-    backup_count = 3
-    format = "{\n\t"time": "%(asctime)s",\n\t"file_name": "%(filename)s",'
-    '\n\t"module": "%(module)s",\n\t"function":"%(funcName)s",\n\t'
-    '"line_number": "%(lineno)s",\n\t"logging_level":'
-    '"%(levelname)s",\n\t"message": "%(message)s"\n}"
-    level = "INFO" # Level should be in CAPS
-    rotate_bytes = 512000
-
-You can use the logger in your code by:
-
-.. code-block:: python
-
-    panaetius.logger.info('some log message')
-
-which gives an output of:
-
-.. code-block:: json
-
-    {
-        "time": "2020-01-13 23:07:17,913",
-        "file_name": "test.py",
-        "module": "test",
-        "function":"<module>",
-        "line_number": "33",
-        "logging_level":"INFO",
-        "message": "some logging message"
-    }
-
-
-Importing and using the api
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-See `panaetius api page`_ on how to use and import the module.
-
-.. _panaetius api page: https://panaetius.readthedocs.io/en/latest/modules/panaetius.html
-
-
-Configuration
----------------
-
-See `configuration page`_ on how to configure ``panaetius``.
-
-.. _configuration page: https://panaetius.readthedocs.io/en/latest/configuration.html

docs/source/modules/panaetius.config.rst

@@ -1,9 +0,0 @@
-.. include:: ../global.rst
-
-panaetius.config :modname:`panaetius.config`
----------------------------------------------
-
-.. automodule:: panaetius.config
-   :members:
-   :undoc-members:
-   :show-inheritance:

docs/source/modules/panaetius.config_inst.rst

@@ -1,9 +0,0 @@
-.. include:: ../global.rst
-
-panaetius.config_inst :modname:`panaetius.config_inst`
---------------------------------------------------------
-
-.. automodule:: panaetius.config_inst
-   :members:
-   :undoc-members:
-   :show-inheritance:

docs/source/modules/panaetius.db.rst

@@ -1,9 +0,0 @@
-.. include:: ../global.rst
-
-panaetius.db :modname:`panaetius.db`
--------------------------------------
-
-.. automodule:: panaetius.db
-   :members:
-   :undoc-members:
-   :show-inheritance:

docs/source/modules/panaetius.header.rst

@@ -1,9 +0,0 @@
-.. include:: ../global.rst
-
-panaetius.header :modname:`panaetius.header`
----------------------------------------------
-
-.. automodule:: panaetius.header
-   :members:
-   :undoc-members:
-   :show-inheritance:

docs/source/modules/panaetius.library.rst

@@ -1,9 +0,0 @@
-.. include:: ../global.rst
-
-panaetius.library :modname:`panaetius.library`
-------------------------------------------------
-
-.. automodule:: panaetius.library
-   :members:
-   :undoc-members:
-   :show-inheritance:

docs/source/modules/panaetius.logging.rst

@@ -1,9 +0,0 @@
-.. include:: ../global.rst
-
-panaetius.logging :modname:`panaetius.logging`
-----------------------------------------------
-
-.. automodule:: panaetius.logging
-   :members:
-   :undoc-members:
-   :show-inheritance:

docs/source/modules/panaetius.rst

@@ -1,66 +0,0 @@
-.. include:: ../global.rst
-
-*********
-panaetius
-*********
-
-API
-===
-
-The following is availble by importing the module:
-
-.. code-block:: python
-
-    import panaetius
-
-
-panaetius.CONFIG
-----------------
-
-:obj:`panaetius.CONFIG` provides an instance of :class:`panaetius.config.Config`
-
-
-panaetius.set_config()
------------------------
-
-Conveniently provides :func:`panaetius.library.set_config` 
-
-Use in your module/script with:
-
-.. code-block:: python
-
-    panaetius.set_config(panaetius.CONFIG, 'aws.secret_key', str, mask=True)    
-
-
-panaetius.CONFIG.aws_secret_key
--------------------------------
-
-Conveniently provides access to all attributes that have been declared with :func:`panaetius.library.set_config`:
-
-.. code-block:: python
-
-    my_secret_key = panaetius.CONFIG.aws_secret_key
-
-
-panaetius.logger
------------------
-
-:obj:`panaetius.logger` provides a logger instance already formatted with a nice json output.
-
-.. code-block:: python
-
-    panaetius.logger.info('some logging message')
-
-This gives a logger output of:
-
-.. code-block:: json
-
-    {
-        "time": "2020-01-13 23:07:17,913",
-        "file_name": "test.py",
-        "module": "test",
-        "function":"<module>",
-        "line_number": "33",
-        "logging_level":"INFO",
-        "message": "some logging message"
-    }

docs/source/toc.rst

@@ -1,27 +0,0 @@
-.. toctree::
-   :maxdepth: 1
-   :caption: Overview
-   :titlesonly:
-
-   introduction
-   configuration
-   changelog
-
-.. toctree::
-   :maxdepth: 4
-   :caption: Modules
-   :titlesonly:
-
-   modules/panaetius.rst
-
-.. toctree::
-   :maxdepth: 4
-   :caption: Submodules
-   :titlesonly:
-
-   modules/panaetius.config.rst
-   modules/panaetius.config_inst.rst
-   modules/panaetius.db.rst
-   modules/panaetius.header.rst
-   modules/panaetius.library.rst
-   modules/panaetius.logging.rst

duties.py

@@ -0,0 +1,377 @@
+from __future__ import annotations
+
+import importlib
+import os
+import pathlib
+import re
+import shutil
+import sys
+from io import StringIO
+from typing import List, Optional, Pattern
+from urllib.request import urlopen
+
+from duty import duty
+
+PACKAGE_NAME = "panaetius"
+REPO_URL = "https://github.com/dtomlinson91/panaetius"
+
+
+@duty(post=["export"])
+def update_deps(ctx, dry: bool = False):
+    """
+    Update the dependencies using Poetry.
+
+    Args:
+        ctx: The context instance (passed automatically).
+        dry (bool, optional) = If True will update the `poetry.lock` without updating the
+            dependencies themselves. Defaults to False.
+
+    Example:
+        `duty update_deps dry=False`
+    """
+    dry_run = "--dry-run" if dry else ""
+    ctx.run(
+        ["poetry", "update", dry_run],
+        title=f"Updating poetry deps {dry_run}",
+    )
+
+
+@duty
+def test(ctx):
+    """
+    Run tests using pytest.
+
+    Args:
+        ctx: The context instance (passed automatically).
+    """
+    pytest_results = ctx.run(["pytest", "-v"], pty=True)
+    print(pytest_results)
+
+
+@duty
+def coverage(ctx):
+    """
+    Generate a coverage report and save to XML and HTML.
+
+    Args:
+        ctx: The context instance (passed automatically).
+
+    Example:
+        `duty coverage`
+    """
+    ctx.run(["coverage", "run", "--source", PACKAGE_NAME, "-m", "pytest"])
+    res = ctx.run(["coverage", "report"], pty=True)
+    print(res)
+    ctx.run(["coverage", "html"])
+    ctx.run(["coverage", "xml"])
+
+
+@duty
+def bump(ctx, version: str = "patch"):
+    """
+    Bump the version using Poetry and update _version.py.
+
+    This duty is ran as part of `duty release`.
+
+    Args:
+        ctx: The context instance (passed automatically).
+        version (str, optional) = poetry version flag. Available options are:
+            patch, minor, major. Defaults to patch.
+
+    Example:
+        `duty bump version=major`
+    """
+
+    # bump with poetry
+    result = ctx.run(["poetry", "version", version])
+    new_version = re.search(r"(?:.*)(?:\s)(\d+\.\d+\.\d+)$", result)
+    print(new_version.group(0))
+
+    # update _version.py
+    version_file = pathlib.Path(PACKAGE_NAME) / "_version.py"
+    with version_file.open("w", encoding="utf-8") as version_file:
+        version_file.write(
+            f'"""Module containing the version of {PACKAGE_NAME}."""\n\n' + f'__version__ = "{new_version.group(1)}"\n'
+        )
+    print(f"Bumped _version.py to {new_version.group(1)}")
+
+
+@duty
+def build(ctx):
+    """
+    Build with poetry and extract the setup.py and copy to project root.
+
+    Args:
+        ctx: The context instance (passed automatically).
+
+    Example:
+        `duty build`
+    """
+
+    repo_root = pathlib.Path(".")
+
+    # build with poetry
+    result = ctx.run(["poetry", "build"])
+    print(result)
+
+    # extract the setup.py from the tar
+    extracted_tar = re.search(r"(?:.*)(?:Built\s)(.*)", result)
+    tar_file = pathlib.Path(f"./dist/{extracted_tar.group(1)}")
+    shutil.unpack_archive(tar_file, tar_file.parents[0])
+
+    # copy setup.py to repo root
+    extracted_path = tar_file.parents[0] / os.path.splitext(tar_file.stem)[0]
+    setup_py = extracted_path / "setup.py"
+    shutil.copyfile(setup_py, (repo_root / "setup.py"))
+
+    # cleanup
+    shutil.rmtree(extracted_path)
+
+
+@duty
+def release(ctx, version: str = "patch") -> None:
+    """
+    Prepare package for a new release.
+
+    Will run bump, build, export. Manual running of publish is required afterwards.
+
+    Args:
+        ctx: The context instance (passed automatically).
+        version (str): poetry version flag. Available options are: patch, minor, major.
+    """
+    print(ctx.run(["duty", "bump", f"version={version}"]))
+    ctx.run(["duty", "build"])
+    ctx.run(["duty", "export"])
+    print(
+        "✔ Check generated files. Run `duty changelog planned_release= previous_release=` and `duty publish password=`"
+        " when ready to publish."
+    )
+
+
+@duty
+def export(ctx):
+    """
+    Export the dependencies to a requirements.txt file.
+
+    Args:
+        ctx: The context instance (passed automatically).
+
+    Example:
+        `duty export`
+    """
+    requirements_content = ctx.run(
+        [
+            "poetry",
+            "export",
+            "-f",
+            "requirements.txt",
+            "--without-hashes",
+        ]
+    )
+    requirements_dev_content = ctx.run(
+        [
+            "poetry",
+            "export",
+            "-f",
+            "requirements.txt",
+            "--without-hashes",
+            "--dev",
+        ]
+    )
+
+    requirements = pathlib.Path(".") / "requirements.txt"
+    requirements_dev = pathlib.Path(".") / "requirements_dev.txt"
+
+    with requirements.open("w", encoding="utf-8") as req:
+        req.write(requirements_content)
+
+    with requirements_dev.open("w", encoding="utf-8") as req:
+        req.write(requirements_dev_content)
+
+
+@duty
+def publish(ctx, password: str):
+    """
+    Publish the package to pypi.org.
+
+    Args:
+        ctx: The context instance (passed automatically).
+        password (str): pypi.org password.
+
+    Example:
+        `duty publish password=$my_password`
+    """
+    dist_dir = pathlib.Path(".") / "dist"
+    rm_result = rm_tree(dist_dir)
+    print(rm_result)
+
+    publish_result = ctx.run(["poetry", "publish", "-u", "dtomlinson", "-p", password, "--build"])
+    print(publish_result)
+
+
+@duty(silent=True)
+def clean(ctx):
+    """
+    Delete temporary files.
+
+    Args:
+        ctx: The context instance (passed automatically).
+    """
+    ctx.run("rm -rf .mypy_cache")
+    ctx.run("rm -rf .pytest_cache")
+    ctx.run("rm -rf tests/.pytest_cache")
+    ctx.run("rm -rf build")
+    ctx.run("rm -rf dist")
+    ctx.run("rm -rf pip-wheel-metadata")
+    ctx.run("rm -rf site")
+    ctx.run("rm -rf coverage.xml")
+    ctx.run("rm -rf pytest.xml")
+    ctx.run("rm -rf htmlcov")
+    ctx.run("find . -iname '.coverage*' -not -name .coveragerc | xargs rm -rf")
+    ctx.run("find . -type d -name __pycache__ | xargs rm -rf")
+    ctx.run("find . -name '*.rej' -delete")
+
+
+@duty
+def format(ctx):
+    """
+    Format code using Black and isort.
+
+    Args:
+        ctx: The context instance (passed automatically).
+    """
+    res = ctx.run(["black", "--line-length=99", PACKAGE_NAME], pty=True, title="Running Black")
+    print(res)
+
+    res = ctx.run(["isort", PACKAGE_NAME])
+    print(res)
+
+
+@duty(pre=["check_code_quality", "check_types", "check_docs", "check_dependencies"])
+def check(ctx):
+    """
+    Check the code quality, check types, check documentation builds and check dependencies for vulnerabilities.
+
+    Args:
+        ctx: The context instance (passed automatically).
+    """
+
+
+@duty
+def check_code_quality(ctx):
+    """
+    Check the code quality using prospector.
+
+    Args:
+        ctx: The context instance (passed automatically).
+    """
+    ctx.run(["prospector", PACKAGE_NAME], pty=True, title="Checking code quality with prospector")
+
+
+@duty
+def check_types(ctx):
+    """
+    Check the types using mypy.
+
+    Args:
+        ctx: The context instance (passed automatically).
+    """
+    ctx.run(["mypy", PACKAGE_NAME], pty=True, title="Checking types with MyPy")
+
+
+@duty
+def check_docs(ctx):
+    """
+    Check the documentation builds successfully.
+
+    Args:
+        ctx: The context instance (passed automatically).
+    """
+    ctx.run(["mkdocs", "build"], title="Building documentation")
+
+
+@duty
+def check_dependencies(ctx):
+    """
+    Check dependencies with safety for vulnerabilities.
+
+    Args:
+        ctx: The context instance (passed automatically).
+    """
+    for module in sys.modules:
+        if module.startswith("safety.") or module == "safety":
+            del sys.modules[module]
+
+    importlib.invalidate_caches()
+
+    from safety import safety
+    from safety.formatter import report
+    from safety.util import read_requirements
+
+    requirements = ctx.run(
+        "poetry export --dev --without-hashes",
+        title="Exporting dependencies as requirements",
+        allow_overrides=False,
+    )
+
+    def check_vulns():
+        packages = list(read_requirements(StringIO(requirements)))
+        vulns = safety.check(packages=packages, ignore_ids="41002", key="", db_mirror="", cached=False, proxy={})
+        output_report = report(vulns=vulns, full=True, checked_packages=len(packages))
+        print(vulns)
+        if vulns:
+            print(output_report)
+
+    ctx.run(
+        check_vulns,
+        stdin=requirements,
+        title="Checking dependencies",
+        pty=True,
+    )
+
+
+@duty
+def changelog(ctx, planned_release: Optional[str] = None, previous_release: Optional[str] = None):
+    """
+    Generate a changelog with git-cliff.
+
+    Args:
+        ctx: The context instance (passed automatically).
+        planned_release (str, optional): The planned release version. Example: v1.0.2
+        previous_release (str, optional): The previous release version. Example: v1.0.1
+    """
+    generated_changelog: str = ctx.run(["git", "cliff", "-u", "-t", planned_release, "-s", "header"])[:-1]
+    if previous_release is not None:
+        generated_changelog: list = generated_changelog.splitlines()
+        generated_changelog.insert(
+            1,
+            f"<small>[Compare with {previous_release}]({REPO_URL}/compare/{previous_release}..{planned_release})</small>",
+        )
+        generated_changelog: str = "\n".join([line for line in generated_changelog]) + "\n"
+    new_changelog = []
+
+    changelog_file = pathlib.Path(".") / "CHANGELOG.md"
+    with changelog_file.open("r", encoding="utf-8") as changelog_contents:
+        all_lines = changelog_contents.readlines()
+        for line_string in all_lines:
+            regex_string = re.search(r"(<!-- marker -->)", line_string)
+            new_changelog.append(line_string)
+            if isinstance(regex_string, re.Match):
+                new_changelog.append(generated_changelog)
+    with changelog_file.open("w", encoding="utf-8") as changelog_contents:
+        changelog_contents.writelines(new_changelog)
+
+
+def rm_tree(directory: pathlib.Path):
+    """
+    Recursively delete a directory and all its contents.
+
+    Args:
+        directory (pathlib.Path): The directory to delete.
+    """
+    for child in directory.glob("*"):
+        if child.is_file():
+            child.unlink()
+        else:
+            rm_tree(child)
+    directory.rmdir()

panaetius/__init__.py

@@ -0,0 +1,11 @@
+"""
+Panaetius package.
+
+A utility library to read variables and provide convenient logging.
+
+Author: Daniel Tomlinson (dtomlinson@panaetius.co.uk)
+"""
+
+from panaetius.config import Config
+from panaetius.library import set_config
+from panaetius.logging import set_logger, SimpleLogger, AdvancedLogger, CustomLogger

panaetius/_version.py

@@ -0,0 +1,3 @@
+"""Module containing the version of panaetius."""
+
+__version__ = "2.3.4"

panaetius/config.py

@@ -0,0 +1,204 @@
+"""
+Config module to access variables from a config file or an environment variable.
+
+This module defines the `Config` class to interact and read variables from either a
+`config.yml` or an environment variable.
+"""
+
+from __future__ import annotations
+
+import ast
+import os
+import pathlib
+from typing import Any
+
+# import toml
+import yaml
+
+from panaetius.exceptions import KeyErrorTooDeepException
+
+
+class Config:
+    """
+    A configuration class to access user variables.
+
+    Args:
+        header_variable (str): the `header` variable.
+        config_path (str|None=None): the path where the header directory is stored.
+        skip_header_init (bool=False): if True will not use a header subdirectory in the
+            `config_path`.
+
+    """
+
+    def __init__(
+        self,
+        header_variable: str,
+        config_path: str | None = None,
+        skip_header_init: bool = False,
+    ) -> None:
+        """
+        Create a Config object to set and access variables.
+
+        Args:
+            header_variable (str): Your header variable name.
+            config_path (str, optional): The path where the header directory is stored.
+                Defaults to None on initialisation.
+            skip_header_init (bool, optional): If True will not use a header
+                subdirectory in the `config_path`. Defaults to False.
+
+        Examples:
+            `config_path` defaults to None on initialisation but will be set to `~/.config`.
+
+            A header of `data_analysis` with a config_path of `~/myapps` will define
+                a config file in `~/myapps/data_analysis/config.yml`.
+        """
+        self.header_variable = header_variable
+        self.config_path = (
+            pathlib.Path(config_path).expanduser()
+            if config_path is not None
+            else pathlib.Path.home() / ".config"
+        )
+        self.skip_header_init = skip_header_init
+        self._missing_config = self._check_config_file_exists()
+
+        # default logging options
+        self.logging_path: str | None = None
+        self.logging_rotate_bytes: int = 0
+        self.logging_backup_count: int = 0
+
+    @property
+    def config(self) -> dict:
+        """
+        Return the contents of the config file.
+
+        If no config file is specified then this returns an empty dictionary.
+
+        Returns:
+            dict: The contents of the config `.yml` loaded as a python dictionary.
+        """
+        if self.skip_header_init:
+            config_file_location = self.config_path / "config.yml"
+        else:
+            config_file_location = self.config_path / self.header_variable / "config.yml"
+        try:
+            with open(config_file_location, "r", encoding="utf-8") as config_file:
+                # return dict(toml.load(config_file))
+                return dict(yaml.load(stream=config_file, Loader=yaml.SafeLoader))
+        except FileNotFoundError:
+            return {}
+
+    def get_value(self, key: str, default: Any) -> Any:
+        """
+        Get the value of a variable from the key name.
+
+        The key can either be one (`value`) or two (`data.value`) levels deep.
+
+        A key of `value` (with a header of `data_analysis`) would refer to a
+        `config.yml` of:
+
+        ```
+        [data_analysis]
+        value = "some value"
+        ```
+
+        or an environment variable of `DATA_ANALYSIS_VALUE="'some value'"`.
+
+        A key of `data.value` would refer to a `config.yml` of:
+        ```
+        [data_analysis.data]
+        value = "some value"
+        ```
+        or an environment variable of `DATA_ANALYSIS_DATA_VALUE="'some value'"`.
+
+        Args:
+            key (str): The key of the variable.
+            default (Any): The default value if the key cannot be found in the config
+                file, or an environment variable.
+
+        Returns:
+            Any: The value of the variable.
+        """
+        env_key = f"{self.header_variable.upper()}_{key.upper().replace('.', '_')}"
+
+        if not self._missing_config:
+            # look in the config file
+            return self._get_config_value(env_key, key, default)
+        # no config file, look for env vars
+        return self._get_env_value(env_key, default)
+
+    def _check_config_file_exists(self) -> bool:
+        if self.skip_header_init is False:
+            config_file_location = self.config_path / self.header_variable / "config.yml"
+        else:
+            config_file_location = self.config_path / "config.yml"
+        try:
+            with open(config_file_location, "r", encoding="utf-8"):
+                return False
+        except FileNotFoundError:
+            return True
+
+    def _get_config_value(self, env_key: str, key: str, default: Any) -> Any:
+        try:
+            # look under top header
+            # REVIEW: could this be auto handled for a key of arbitrary length?
+
+            # check for env variable and have it take priority
+            value = os.environ.get(env_key.replace("-", "_"))
+            if value is not None:
+                return self.__get_config_value_env_var_override(value)
+
+            if len(key.split(".")) > 3:
+                raise KeyErrorTooDeepException(
+                    f"Your key of {key} can only be 3 levels deep maximum."
+                )
+            if len(key.split(".")) == 1:
+                return self.__get_config_value_key_split_once(key)
+            if len(key.split(".")) == 2:
+                return self.__get_config_value_key_split_twice(key)
+            if len(key.split(".")) == 3:
+                return self.__get_config_value_key_split_thrice(key)
+            raise KeyError()
+
+        except (KeyError, TypeError):
+            if value is None:
+                return self.__get_config_value_missing_key_value_is_none(default)
+            # if env var is present, load it
+            return self.__get_config_value_missing_key_value_is_not_none(value)
+
+    def __get_config_value_key_split_once(self, key: str) -> Any:
+        name = key.lower()
+        return self.config[self.header_variable][name]
+
+    def __get_config_value_key_split_twice(self, key: str) -> Any:
+        section, name = key.lower().split(".")
+        return self.config[self.header_variable][section][name]
+
+    def __get_config_value_key_split_thrice(self, key: str) -> Any:
+        section, name_0, name_1 = key.lower().split(".")
+        return self.config[self.header_variable][section][name_0][name_1]
+
+    def __get_config_value_missing_key_value_is_none(self, default: Any) -> Any:
+        return self.__load_default_value(default)
+
+    def __get_config_value_missing_key_value_is_not_none(self, value: str) -> Any:
+        return self.__load_value(value)
+
+    def __get_config_value_env_var_override(self, value: str) -> Any:
+        return self.__load_value(value)
+
+    def _get_env_value(self, env_key: str, default: Any) -> Any:  # noqa
+        # look for an environment variable, fallback to default
+        value = os.environ.get(env_key.replace("-", "_"))
+        if value is None:
+            return self.__load_default_value(default)
+        return self.__load_value(value)
+
+    def __load_value(self, value: str) -> Any:  # noqa
+        try:
+            return ast.literal_eval(value)
+        except (ValueError, SyntaxError):
+            # string without spaces: ValueError, with spaces; SyntaxError
+            return value
+
+    def __load_default_value(self, default: Any) -> Any:  # noqa
+        return default

panaetius/exceptions.py

@@ -0,0 +1,13 @@
+"""Module that defines custom exceptions for Panetius."""
+
+
+class KeyErrorTooDeepException(Exception):
+    """Raised if the keys in the config.yml are nested too deeply."""
+
+
+class LoggingDirectoryDoesNotExistException(Exception):
+    """Raised if the logging directory does not exist."""
+
+
+class InvalidPythonException(Exception):
+    """Raised if the environement variable Python type is invalid."""

panaetius/library.py

@@ -0,0 +1,43 @@
+"""Module to provide functionality when interacting with variables."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from panaetius import Config
+
+
+def set_config(
+    config_inst: Config,
+    key: str,
+    default: Any = None,
+) -> None:
+    """
+    Define a variable to be read from a `config.toml` or an environment variable.
+
+    Args:
+        config_inst (Config): The instance of the `Config` class.
+        key (str): The key of the variable.
+        default (Any, optional): The default value if the key cannot be found in the config
+            file, or an environment variable. Defaults to None.
+
+    Example:
+        `set_config(CONFIG, "value", default=[1, 2])` would look for a
+        `config.toml` with the following structure (with `CONFIG` having a header of
+        `data_analysis`):
+
+        ```
+        [data_analysis]
+        value = "some value"
+        ```
+
+        Or an environment variable of `DATA_ANALYSIS_VALUE="'some value'"`.
+
+        If found, this value can be access with `CONFIG.value` which would return
+        `some_value`.
+
+        If neither the environment variable nor the `config.toml` are present, the
+        default of `[1, 2]` would be returned instead.
+    """
+    config_var = key.lower().replace(".", "_")
+    setattr(config_inst, config_var, config_inst.get_value(key, default))

panaetius/logging.py

@@ -0,0 +1,164 @@
+"""Module to define a convenient logger instance with json formatted output."""
+
+from __future__ import annotations
+
+from abc import ABCMeta, abstractmethod
+import logging
+from logging.handlers import RotatingFileHandler
+import pathlib
+import sys
+
+from panaetius import Config
+from panaetius.library import set_config
+from panaetius.exceptions import LoggingDirectoryDoesNotExistException
+
+
+def set_logger(config_inst: Config, logging_format_inst: LoggingData) -> logging.Logger:
+    """
+    Set and return a `logging.Logger` instance for quick logging.
+
+    `logging_format_inst` should be an instance of either SimpleLogger, AdvancedLogger,
+    or CustomLogger.
+
+    SimpleLogger and AdvancedLogger define a logging format and a logging level info.
+
+    CustomLogger defines a logging level info and should have a logging format passed
+    in.
+
+    Logging to a file is defined by a `logging.path` key set on `Config`. This path
+    should exist as it will not be created.
+
+    Args:
+        config_inst (Config): The instance of the `Config` class.
+        logging_format_inst (LoggingData): The instance of the `LoggingData` class.
+
+    Raises:
+        LoggingDirectoryDoesNotExistException: If the logging directory specified does
+            not exist.
+
+    Returns:
+        logging.Logger: An configured instance of `logging.Logger` ready to be used.
+
+    Example:
+
+        ```
+        logger = set_logger(CONFIG, SimpleLogger())
+
+        logger.info("some logging message")
+        ```
+
+        Would create a logging output of:
+
+        ```
+        {
+            "time": "2021-10-18 02:26:24,037",
+            "logging_level":"INFO",
+            "message": "some logging message"
+        }
+        ```
+
+    """
+    logger = logging.getLogger(config_inst.header_variable)
+    log_handler_sys = logging.StreamHandler(sys.stdout)
+
+    # configure file handler
+    if config_inst.logging_path is not None:
+        if not config_inst.skip_header_init:
+            logging_file = (
+                pathlib.Path(config_inst.logging_path)
+                / config_inst.header_variable
+                / f"{config_inst.header_variable}.log"
+            ).expanduser()
+        else:
+            logging_file = (
+                pathlib.Path(config_inst.logging_path)
+                / f"{config_inst.header_variable}.log"
+            ).expanduser()
+
+        if not logging_file.parents[0].exists():
+            raise LoggingDirectoryDoesNotExistException()
+
+        if config_inst.logging_rotate_bytes == 0:
+            set_config(config_inst, "logging.rotate_bytes", 512000)
+        if config_inst.logging_backup_count == 0:
+            set_config(config_inst, "logging.backup_count", 3)
+
+        log_handler_file = RotatingFileHandler(
+            str(logging_file),
+            "a",
+            config_inst.logging_rotate_bytes,
+            config_inst.logging_backup_count,
+        )
+
+        log_handler_file.setFormatter(logging.Formatter(logging_format_inst.format))
+        logger.addHandler(log_handler_file)
+
+    # configure stdout handler
+    log_handler_sys.setFormatter(logging.Formatter(logging_format_inst.format))
+    logger.addHandler(log_handler_sys)
+    logger.setLevel(logging_format_inst.logging_level)
+    return logger
+
+
+class LoggingData(metaclass=ABCMeta):
+    @property
+    @abstractmethod
+    def format(self) -> str:
+        raise NotImplementedError
+
+    @property
+    @abstractmethod
+    def logging_level(self) -> str:
+        raise NotImplementedError
+
+    @abstractmethod
+    def __init__(self, logging_level: str):
+        raise NotImplementedError
+
+
+class SimpleLogger(LoggingData):
+    @property
+    def format(self) -> str:
+        return str(
+            '{\n\t"time": "%(asctime)s",\n\t"logging_level":'
+            '"%(levelname)s",\n\t"message": "%(message)s"\n}',
+        )
+
+    @property
+    def logging_level(self) -> str:
+        return self._logging_level
+
+    def __init__(self, logging_level: str = "INFO"):
+        self._logging_level = logging_level
+
+
+class AdvancedLogger(LoggingData):
+    @property
+    def format(self) -> str:
+        return str(
+            '{\n\t"time": "%(asctime)s",\n\t"file_name": "%(filename)s",'
+            '\n\t"module": "%(module)s",\n\t"function":"%(funcName)s",\n\t'
+            '"line_number": "%(lineno)s",\n\t"logging_level":'
+            '"%(levelname)s",\n\t"message": "%(message)s"\n}',
+        )
+
+    @property
+    def logging_level(self) -> str:
+        return self._logging_level
+
+    def __init__(self, logging_level: str = "INFO"):
+        self._logging_level = logging_level
+
+
+class CustomLogger(LoggingData):
+    @property
+    def format(self) -> str:
+        return str(self._format)
+
+    @property
+    def logging_level(self) -> str:
+        return self._logging_level
+
+    def __init__(self, logging_format: str, logging_level: str = "INFO"):
+        self._logging_level = logging_level
+        self._format = logging_format

panaetius/utilities/__init__.py

@@ -0,0 +1,3 @@
+"""Sub-package which defines general utility functions."""
+
+from panaetius.utilities.squasher import Squash

panaetius/utilities/squasher.py

@@ -0,0 +1,64 @@
+"""Sub-module that defines squashing json objects into a single json object."""
+
+from __future__ import annotations
+
+from copy import deepcopy
+import itertools
+from typing import Iterator, Tuple
+
+
+class Squash:
+    """Squash a json object or Python dictionary into a single level dictionary."""
+
+    def __init__(self, data: dict) -> None:
+        """
+        Create a Squash object to squash data into a single level dictionary.
+
+        Args:
+            data (dict): [description]
+
+        Example:
+            squashed_data = Squash(my_data)
+
+            squashed_data.as_dict
+        """
+        self.data = data
+
+    @property
+    def as_dict(self) -> dict:
+        """
+        Return the squashed data as a dictionary.
+
+        Returns:
+            dict: The original data squashed as a dict.
+        """
+        return self._squash()
+
+    @staticmethod
+    def _unpack_dict(
+        key: str, value: dict | list | str
+    ) -> Iterator[Tuple[str, dict | list | str]]:
+        if isinstance(value, dict):
+            for sub_key, sub_value in value.items():
+                temporary_key = f"{key}_{sub_key}"
+                yield temporary_key, sub_value
+        elif isinstance(value, list):
+            for index, sub_value in enumerate(value):
+                temporary_key = f"{key}_{index}"
+                yield temporary_key, sub_value
+        else:
+            yield key, value
+
+    def _squash(self) -> dict:
+        result = deepcopy(self.data)
+        while True:
+            result = dict(
+                itertools.chain.from_iterable(
+                    itertools.starmap(self._unpack_dict, result.items())
+                )
+            )
+            if not any(
+                isinstance(value, dict) for value in result.values()
+            ) and not any(isinstance(value, list) for value in result.values()):
+                break
+        return result

prospector.yaml

@@ -0,0 +1,117 @@
+output-format: vscode
+doc-warnings: true
+strictness: none
+
+ignore-patterns:
+  - (^|/)\..+
+
+# https://pylint.pycqa.org/en/latest/technical_reference/features.html
+pylint:
+  run: true
+  disable:
+    # disables TODO warnings
+    - fixme
+# !doc docstrings
+    # - missing-module-docstring
+    # - missing-class-docstring
+    # - missing-function-docstring
+# ! doc end of docstrings
+    # disables warnings about abstract methods not overridden
+    - abstract-method
+    # used when an ancestor class method has an __init__ method which is not called by a derived class.
+    - super-init-not-called
+    # either all return statements in a function should return an expression, or none of them should.
+    # - inconsistent-return-statements
+    # Used when an expression that is not a function call is assigned to nothing. Probably something else was intended.
+    # - expression-not-assigned
+    # Used when a line is longer than a given number of characters.
+    - line-too-long
+  enable:
+  options:
+    max-locals: 15
+    max-returns: 6
+    max-branches: 12
+    max-statements: 50
+    max-parents: 7
+    max-attributes: 20
+    min-public-methods: 0
+    max-public-methods: 25
+    max-module-lines: 1000
+    max-line-length: 88
+    max-args: 8
+
+mccabe:
+  run: true
+  options:
+    max-complexity: 10
+
+# https://pep8.readthedocs.io/en/release-1.7.x/intro.html#error-codes
+pep8:
+  run: true
+  options:
+    max-line-length: 88
+    single-line-if-stmt: n
+  disable:
+    # line too long
+    - E501
+
+pyroma:
+  run: false
+  disable:
+    - PYR19
+    - PYR16
+
+# https://pep257.readthedocs.io/en/latest/error_codes.html
+# http://www.pydocstyle.org/en/6.1.1/error_codes.html
+pep257:
+  disable:
+# !doc docstrings
+    # Missing docstring in __init__
+    # - D107
+    # Missing docstring in public module
+    # - D100
+    # Missing docstring in public class
+    # - D101
+    # Missing docstring in public method
+    # - D102
+    # Missing docstring in public function
+    # - D103
+    # Multi-line docstring summary should start at the second line
+    # - D213
+    # First word of the docstring should not be This
+    # - D404
+    # DEFAULT IGNORES
+    # 1 blank line required before class docstring
+    - D203
+    # Multi-line docstring summary should start at the first line
+    - D212
+# !doc end of docstrings
+    # Section name should end with a newline
+    - D406
+    # Missing dashed underline after section
+    - D407
+    # Missing blank line after last section
+    - D413
+
+# https://flake8.pycqa.org/en/latest/user/error-codes.html
+pyflakes:
+  disable:
+    # module imported but unused
+    - F401
+
+dodgy:
+  run: true
+
+bandit:
+  run: true
+  # options:
+    # ignore assert warning
+    # - B101
+
+mypy:
+  run: true
+  options:
+    # https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-type-hints-for-third-party-library
+    ignore-missing-imports: true
+    # https://mypy.readthedocs.io/en/stable/running_mypy.html#following-imports
+    follow-imports: normal

pyproject.toml

@@ -1,10 +1,10 @@
 [tool.poetry]
 name = "panaetius"
-version = "1.0.2"
+version = "2.3.4"
 description = "Python module to gracefully handle a .config file/environment variables for scripts, with built in masking for sensitive options. Provides a Splunk friendly formatted logger instance."
 license = "MIT"
 authors = ["dtomlinson <dtomlinson@panaetius.co.uk>"]
-readme = "/Users/dtomlinson/git-repos/projects/panaetius/README.rst"
+readme = "./README.md"
 homepage = "https://github.com/dtomlinson91/panaetius"
 repository = "https://github.com/dtomlinson91/panaetius"
 documentation = "https://panaetius.readthedocs.io/en/latest/introduction.html"
@@ -25,21 +25,33 @@ classifiers = [
 [tool.poetry.dependencies]
 python = "^3.7"
 toml = "^0.10.0"
-pylite = "^0.1.0"
+PyYAML = "^6.0"
 
 [tool.poetry.dev-dependencies]
-pytest = "^3.0"
-autopep8 = "^1.4"
-pudb = "^2019.2"
-McCabe = "^0.6.1"
-YAPF = "^0.29.0"
-pydocstyle = "^5.0"
-Pyflakes = "^2.1"
-Rope = "^0.16.0"
-python-language-server = "^0.31.4"
-pycodestyle = "^2.5"
-sphinx = "^2.3"
-sphinx_rtd_theme = "^0.4.3"
+prospector = {extras = ["with_bandit", "with_mypy"], version = "^1.5.1"}
+types-toml = "^0.10.1"
+pytest = "^6.2.5"
+pytest-datadir = "^1.3.1"
+pytest-xdist = "^2.4.0"
+coverage = "^6.0.2"
+duty = "^0.7.0"
+types-PyYAML = "^6.0.1"
+isort = "^5.10.1"
+mypy = "^0.910"
+safety = "^1.10.3"
+
+[tool.black]
+line-length = 120
+
+[tool.isort]
+line-length = 120
+not_skip = "__init__.py"
+multi_line_output = 3
+force_single_line = false
+balanced_wrapping = true
+default_section = "THIRDPARTY"
+known_first_party = "duty"
+include_trailing_comma = true
 
 [build-system]
 requires = ["poetry>=0.12"]

pytest.ini

@@ -0,0 +1,3 @@
+; ; parallel tests with pytest-xdist
+; [pytest]
+; addopts=-n4

requirements.txt

@@ -1,7 +1,2 @@
-pylite==0.1.0 \
-    --hash=sha256:e338d20d3f8f72dd84d1e58f2fd6dba008d593e0cfacfb5fbdd5a297b830628e \
-    --hash=sha256:eb46f5beb1f2102672fd4355c013ac2feebc0df284d65f7711f2041a0a410141
-toml==0.10.0 \
-    --hash=sha256:229f81c57791a41d65e399fc06bf0848bab550a9dfd5ed66df18ce5f05e73d5c \
-    --hash=sha256:235682dd292d5899d361a811df37e04a8828a5b1da3115886b73cf81ebc9100e \
-    --hash=sha256:f1db651f9657708513243e61e6cc67d101a39bad662eaa9b5546f789338e07a3
+pyyaml==6.0; python_version >= "3.6"
+toml==0.10.2; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.3.0")

requirements_dev.txt

@@ -0,0 +1,68 @@
+ansimarkup==1.5.0; python_version >= "3.6"
+astroid==2.9.1; python_full_version >= "3.6.2" and python_version < "4.0"
+atomicwrites==1.4.0; python_version >= "3.6" and python_full_version < "3.0.0" and sys_platform == "win32" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6") or sys_platform == "win32" and python_version >= "3.6" and python_full_version >= "3.4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6")
+attrs==21.4.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
+bandit==1.7.1; python_full_version >= "3.6.1" and python_version < "4.0" and python_version >= "3.5"
+cached-property==1.5.2; python_version < "3.8" and python_version >= "3.6"
+certifi==2021.10.8; python_version >= "3.5" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.5"
+charset-normalizer==2.0.9; python_full_version >= "3.6.0" and python_version >= "3.5"
+click==8.0.3; python_version >= "3.6"
+colorama==0.4.4; sys_platform == "win32" and python_version >= "3.6" and python_full_version >= "3.6.2" and python_version < "4.0" and platform_system == "Windows" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6")
+coverage==6.2; python_version >= "3.6"
+dodgy==0.2.1; python_full_version >= "3.6.1" and python_version < "4.0"
+dparse==0.5.1; python_version >= "3.5"
+duty==0.7.0; python_version >= "3.6"
+execnet==1.9.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
+failprint==0.8.0; python_version >= "3.6"
+flake8-polyfill==1.0.2; python_full_version >= "3.6.1" and python_version < "4.0"
+flake8==2.3.0; python_full_version >= "3.6.1" and python_version < "4.0"
+gitdb==4.0.9; python_full_version >= "3.6.1" and python_version < "4.0" and python_version >= "3.7"
+gitpython==3.1.24; python_full_version >= "3.6.1" and python_version < "4.0" and python_version >= "3.7"
+idna==3.3; python_version >= "3.5" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.5"
+importlib-metadata==4.10.0; python_version < "3.8" and python_version >= "3.7" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6") and python_full_version >= "3.6.1"
+iniconfig==1.1.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
+isort==5.10.1; python_full_version >= "3.6.1" and python_version < "4.0"
+jinja2==3.0.3; python_version >= "3.6"
+lazy-object-proxy==1.7.1; python_full_version >= "3.6.2" and python_version < "4.0" and python_version >= "3.6"
+markupsafe==2.0.1; python_version >= "3.6"
+mccabe==0.6.1; python_full_version >= "3.6.2" and python_version < "4.0"
+mypy-extensions==0.4.3; python_full_version >= "3.6.1" and python_version < "4.0" and python_version >= "3.5"
+mypy==0.910; python_version >= "3.5"
+packaging==21.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
+pbr==5.8.0; python_full_version >= "3.6.1" and python_version < "4.0" and python_version >= "3.6"
+pep8-naming==0.10.0; python_full_version >= "3.6.1" and python_version < "4.0"
+pep8==1.7.1; python_full_version >= "3.6.1" and python_version < "4.0"
+platformdirs==2.4.1; python_full_version >= "3.6.2" and python_version < "4.0" and python_version >= "3.7"
+pluggy==1.0.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
+prospector==1.5.3.1; python_full_version >= "3.6.1" and python_version < "4.0"
+ptyprocess==0.7.0; sys_platform != "win32" and python_version >= "3.6"
+py==1.11.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
+pycodestyle==2.8.0; python_full_version >= "3.6.1" and python_version < "4.0"
+pydocstyle==6.1.1; python_full_version >= "3.6.1" and python_version < "4.0" and python_version >= "3.6"
+pyflakes==2.3.1; python_full_version >= "3.6.1" and python_version < "4.0"
+pylint-celery==0.3; python_full_version >= "3.6.1" and python_version < "4.0"
+pylint-django==2.4.4; python_full_version >= "3.6.1" and python_version < "4.0"
+pylint-flask==0.6; python_full_version >= "3.6.1" and python_version < "4.0"
+pylint-plugin-utils==0.6; python_full_version >= "3.6.1" and python_version < "4.0"
+pylint==2.12.2; python_full_version >= "3.6.2" and python_version < "4.0"
+pyparsing==3.0.6; python_version >= "3.6"
+pytest-datadir==1.3.1; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0")
+pytest-forked==1.4.0; python_version >= "3.6"
+pytest-xdist==2.5.0; python_version >= "3.6"
+pytest==6.2.5; python_version >= "3.6"
+pyyaml==6.0; python_version >= "3.6"
+requests==2.26.0; python_version >= "3.5" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.5"
+requirements-detector==0.7; python_full_version >= "3.6.1" and python_version < "4.0"
+safety==1.10.3; python_version >= "3.5"
+setoptconf-tmp==0.3.1; python_full_version >= "3.6.1" and python_version < "4.0"
+smmap==5.0.0; python_full_version >= "3.6.1" and python_version < "4.0" and python_version >= "3.7"
+snowballstemmer==2.2.0; python_full_version >= "3.6.1" and python_version < "4.0" and python_version >= "3.6"
+stevedore==3.5.0; python_full_version >= "3.6.1" and python_version < "4.0" and python_version >= "3.6"
+toml==0.10.2; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.3.0")
+typed-ast==1.4.3; python_version < "3.8" and python_version >= "3.5" and python_full_version >= "3.6.2" and implementation_name == "cpython"
+types-pyyaml==6.0.1
+types-toml==0.10.1
+typing-extensions==4.0.1; python_full_version >= "3.6.2" and python_version < "3.8" and python_version >= "3.7"
+urllib3==1.26.7; python_version >= "3.5" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version < "4" and python_version >= "3.5"
+wrapt==1.13.3; python_full_version >= "3.6.2" and python_version < "4.0"
+zipp==3.7.0; python_version < "3.8" and python_version >= "3.7"

setup.py

@@ -1,27 +1,25 @@
 # -*- coding: utf-8 -*-
-from distutils.core import setup
-
-package_dir = \
-{'': 'src'}
+from setuptools import setup
 
 packages = \
-['panaetius']
+['panaetius', 'panaetius.utilities']
 
 package_data = \
 {'': ['*']}
 
 install_requires = \
-['pylite>=0.1.0,<0.2.0', 'toml>=0.10.0,<0.11.0']
+['PyYAML>=6.0,<7.0', 'toml>=0.10.0,<0.11.0']
 
 setup_kwargs = {
     'name': 'panaetius',
-    'version': '1.0.2',
-    'description': 'Python module to gracefully handle a .config file/environment variables for scripts, with built in masking for sensitive options. Provides a Splunk friendly logger instance.',
-    'long_description': 'Author\n=======\n\nDaniel Tomlinson (dtomlinson@panaetius.co.uk)\n\nRequires\n=========\n\n`>= python3.7`\n\nPython requirements\n====================\n\n- toml = "^0.10.0"\n- pylite = "^0.1.0"\n\nDocumentation\n==============\n\nRead the documentation on `read the docs`_.\n\n.. _read the docs: https://panaetius.readthedocs.io/en/latest/introduction.html\n\nInstallation\n==============\n\nYou can install ..:obj:`panaetius`\n\nEasy Way\n=========\n\nPython\n-------\n\nFrom pip\n~~~~~~~~~\n\nFrom local wheel\n~~~~~~~~~~~~~~~~~\n\nFrom source\n~~~~~~~~~~~~\n\nExample Usage\n==============\n\n',
+    'version': '2.3.4',
+    'description': 'Python module to gracefully handle a .config file/environment variables for scripts, with built in masking for sensitive options. Provides a Splunk friendly formatted logger instance.',
+    'long_description': '# Panaetius\n\nThis package provides:\n\n- Functionality to read user variables from a `config.yml` or environment variables.\n- A convenient default logging formatter printing `json` that can save to disk and rotate.\n- Utility functions.\n\n## Config\n\n### options\n\n#### skip_header_init\n\nIf `skip_header_init=True` then the `config_path` will not use the `header_variable` as the\nsub-directory in the `config_path`.\n\nE.g\n\n`CONFIG = panaetius.Config("tembo", "~/tembo/.config", skip_header_init=True)`\n\nWill look in `~/tembo/config/config.yml`.\n\nIf `skip_header_init=False` then would look in `~/tembo/config/tembo/config.yml`.\n\n### Module\n\nConvenient to place in a package/sub-package `__init__.py`.\n\nSee Tembo for an example: <https://github.com/tembo-pages/tembo-core/blob/main/tembo/cli/__init__.py>\n\nExample snippet to use in a module:\n\n```python\n"""Subpackage that contains the CLI application."""\n\nimport os\nfrom typing import Any\n\nimport panaetius\nfrom panaetius.exceptions import LoggingDirectoryDoesNotExistException\n\n\nif (config_path := os.environ.get("TEMBO_CONFIG")) is not None:\n    CONFIG: Any = panaetius.Config("tembo", config_path, skip_header_init=True)\nelse:\n    CONFIG = panaetius.Config(\n        "tembo", "~/tembo/.config", skip_header_init=True\n    )\n\n\npanaetius.set_config(CONFIG, "base_path", "~/tembo")\npanaetius.set_config(CONFIG, "template_path", "~/tembo/.templates")\npanaetius.set_config(CONFIG, "scopes", {})\npanaetius.set_config(CONFIG, "logging.level", "DEBUG")\npanaetius.set_config(CONFIG, "logging.path")\n\ntry:\n    logger = panaetius.set_logger(\n        CONFIG, panaetius.SimpleLogger(logging_level=CONFIG.logging_level)\n    )\nexcept LoggingDirectoryDoesNotExistException:\n    _LOGGING_PATH = CONFIG.logging_path\n    CONFIG.logging_path = ""\n    logger = panaetius.set_logger(\n        CONFIG, panaetius.SimpleLogger(logging_level=CONFIG.logging_level)\n    )\n    logger.warning("Logging directory %s does not exist", _LOGGING_PATH)\n\n```\n\nThis means in `./tembo/cli/cli.py` you can\n\n```python\nimport tembo.cli\n\n# access the CONFIG instance + variables from the config.yml\ntembo.cli.CONFIG\n```\n\n### Script\n\nCreate `./config/config.yml` in the same directory as the script.\n\nIn the script initialise a `CONFIG` object:\n\n```python\nimport pathlib\n\nimport panaetius\n\nCONFIG = panaetius.Config(\n    "teenagers_scraper", str(pathlib.Path(__file__).parents[0] / ".config"), skip_header_init=True\n)\n```\n\nSet variables in the same way as the module above.\n\n#### quickstart logging\n\n```python\nimport panaetius\n\n\ndef get_logger():\n    logging_dir = pathlib.Path(__file__).parents[0] / "logs"\n    logging_dir.mkdir(parents=True, exist_ok=True)\n\n    CONFIG = panaetius.Config("training_data_into_gcp", skip_header_init=True)\n    panaetius.set_config(CONFIG, "logging.level", "DEBUG")\n    panaetius.set_config(CONFIG, "logging.path", logging_dir)\n    return panaetius.set_logger(CONFIG, panaetius.SimpleLogger(logging_level=CONFIG.logging_level))\n```\n\n## Utility Functions\n\n### Squasher\n\nSquashes a json object or Python dictionary into a single level dictionary.\n',
     'author': 'dtomlinson',
     'author_email': 'dtomlinson@panaetius.co.uk',
+    'maintainer': None,
+    'maintainer_email': None,
     'url': 'https://github.com/dtomlinson91/panaetius',
-    'package_dir': package_dir,
     'packages': packages,
     'package_data': package_data,
     'install_requires': install_requires,

src/panaetius/__header__.py

@@ -1 +0,0 @@
-__header__ = 'panaetius_test'

src/panaetius/__init__.py

@@ -1,6 +0,0 @@
-from panaetius.config_inst import CONFIG
-from .config import Config
-from .library import set_config
-from panaetius.header import __header__
-import panaetius.logging
-from panaetius.logging import logger as logger

src/panaetius/__version__.py

@@ -1 +0,0 @@
-__version__ = '1.0.1'

src/panaetius/config.py

@@ -1,206 +0,0 @@
-from typing import Callable, Union
-import os
-import toml
-
-from panaetius.library import export
-from panaetius.header import __header__
-from panaetius.db import Mask
-
-# __all__ = ['Config']
-
-
-@export
-class Config:
-
-    """Handles the config options for the module and stores config variables
-    to be shared.
-
-    Attributes
-    ----------
-    config_file : dict
-        Contains the config options. See
-        :meth:`~panaetius.config.Config.read_config`
-        for the data structure.
-    deferred_messages : list
-        A list containing the messages to be logged once the logger has been
-        instantiated.
-    Mask : panaetius.db.Mask
-        Class to mask values in a config file.
-    module_name : str
-        A string representing the module name. This is added in front of all
-        envrionment variables and is the title of the `config.toml`.
-    path : str
-        Path to config file
-
-    Parameters
-    ----------
-    path : str
-        Path to config file
-    """
-
-    def __init__(self, path: str) -> None:
-        """
-        See :class:`~panaetius.config.Config` for parameters.
-        """
-        self.path = os.path.expanduser(path)
-        self.deferred_messages = []
-        self.config_file = self.read_config(path)
-        self.module_name = __header__.lower()
-        self.Mask = Mask
-
-    def read_config(self, path: str, write: bool = False) -> Union[dict, None]:
-        """Reads the toml config file from `path` if it exists.
-
-        Parameters
-        ----------
-        path : str
-            Path to config file. Should not contain `config.toml`
-
-            Example: ``path = '~/.config/panaetius'``
-
-        Returns
-        -------
-        Union[dict, None]
-            Returns a dict if the file is found else returns nothing.
-
-            The dict contains a key for each header. Each key corresponds to a
-            dictionary containing a key, value pair for each config under
-            that header.
-
-            Example::
-
-                [panaetius]
-
-                [panaetius.foo]
-                foo = bar
-
-            Returns a dict:
-
-                ``{'panaetius' : {foo: {'foo': 'bar'}}}``
-        """
-
-        path += 'config.toml' if path[-1] == '/' else '/config.toml'
-        path = os.path.expanduser(path)
-        if not write:
-            try:
-                with open(path, 'r+') as config_file:
-                    config_file = toml.load(config_file)
-                self.defer_log(f'Config file found at {path}')
-                return config_file
-            except FileNotFoundError:
-                self.defer_log(f'Config file not found at {path}')
-                pass
-        else:
-            try:
-                with open(path, 'w+') as config_file:
-                    config_file = toml.load(config_file)
-                self.defer_log(f'Config file found at {path}')
-                return config_file
-            except FileNotFoundError:
-                self.defer_log(f'Config file not found at {path}')
-                pass
-
-    def get(
-        self,
-        key: str,
-        default: str = None,
-        cast: Callable = None,
-        mask: bool = False,
-    ) -> Union[str, None]:
-        """Retrives the config variable from either the `config.toml` or an
-        environment variable. Will default to the default value if nothing
-        is found
-
-        Parameters
-        ----------
-        key : str
-            Key to the configuration variable. Should be in the form
-            `panaetius.variable` or `panaetius.header.variable`.
-            When loaded, it will be accessable at
-            `Config.panaetius_variable` or
-            `Config.panaetius_header_variable`.
-        default : str, optional
-            The default value if nothing is found. Defaults to `None`.
-        cast : Callable, optional
-            The type of the variable. E.g `int` or `float`. Should reference
-            the type object and not as string. Defaults to `None`.
-
-        Returns
-        -------
-        Any
-            Will return the config variable if found, or the default.
-        """
-        env_key = f"{__header__.upper()}_{key.upper().replace('.', '_')}"
-
-        try:
-            # look in the config.toml
-            if len(key.split('.')) == 2:
-                # look for subsections
-                # print(mask)
-                if mask:
-                    # print('mask', key)
-                    value = self.Mask(
-                        self.path, self.config_file, key
-                    ).get_value()
-                else:
-                    # print('no-mask')
-                    section, name = key.lower().split('.')
-                    value = self.config_file[self.module_name][section][name]
-                    self.defer_log(f'{env_key} found in config.toml')
-            else:
-                # print('valueerror')
-                # look under top level module __header__
-                # key = f'{self.module_name}.key'
-                if mask:
-                    # key = f'{__header__}.{key}'
-                    # print(f'mask key={key}')
-                    value = self.Mask(
-                        self.path, self.config_file, key
-                    ).get_value()
-                else:
-                    name = key.lower()
-                    value = self.config_file[self.module_name][name]
-                    self.defer_log(f'{env_key} found in config.toml')
-            # finally:
-            try:
-                # return if found in config.toml
-                return cast(value) if cast else value
-            except UnboundLocalError:
-                # pass if nothing was found
-                # print('unbound error')
-                pass
-        except KeyError:
-            # print('key error')
-            self.defer_log(f'{env_key} not found in config.toml')
-        except TypeError:
-            # print('type error')
-            self.defer_log(f'{env_key} not found in config.toml')
-
-        # look for an environment variable
-        value = os.environ.get(env_key)
-
-        if value is not None:
-            self.defer_log(f'{env_key} found in an environment variable')
-        else:
-            # fall back to default
-            self.defer_log(f'{env_key} not found in an environment variable.')
-            value = default
-            self.defer_log(f'{env_key} set to default {default}')
-        return cast(value) if cast else value
-
-    def defer_log(self, msg: str) -> None:
-        """Populates a list `Config.deferred_messages` with all the events to
-        be passed to the logger later if required.
-
-        Parameters
-        ----------
-        msg : str
-            The message to be logged.
-        """
-        self.deferred_messages.append(msg)
-
-    def reset_log(self) -> None:
-        """Empties the list `Config.deferred_messages`.
-        """
-        del self.deferred_messages
-        self.deferred_messages = []

src/panaetius/config_inst.py

@@ -1,11 +0,0 @@
-import os
-
-from panaetius.header import __header__
-from panaetius.config import Config
-
-
-DEFAULT_CONFIG_PATH = f'~/.config/{__header__.lower()}'
-CONFIG_PATH = os.environ.get(
-    f'{__header__.upper()}_CONFIG_PATH', DEFAULT_CONFIG_PATH
-)
-CONFIG = Config(CONFIG_PATH)

src/panaetius/db.py

@@ -1,256 +0,0 @@
-from os import path, urandom
-import hashlib
-from typing import Tuple
-import toml
-import io
-
-from pylite.simplite import Pylite
-
-from panaetius.header import __header__ as __header__
-import panaetius
-
-
-class Mask:
-
-    """Class to handle masking sensitive values in a config file
-
-    Attributes
-    ----------
-    config_contents : dict
-        A dict containing the contents of the config file.
-    config_path : str
-        The path to the config file.
-    config_var : str
-        The key corresponding to the config entry.
-    database : Pylite
-        A Pylite instance for the datbase.
-    entry : str
-        The result from the config file. Could either be a hash or the raw
-        value.
-    header : str
-        The __header__ which denotes where the config file is stored.
-    name : str
-        The key of the entry in the config file.
-    result : str
-        The value of the entry in the config file.
-    table_name : str
-        The sqlite table name. Defaults to the __header__ value.
-    """
-
-    @property
-    def hash(self):
-        """Property to determine the hash of a config entry.
-
-        Returns
-        -------
-        bytes
-            The hash as a bytes object.
-        """
-        try:
-            if not self._hash_exists:
-                pass
-        except AttributeError:
-            self._hash = hashlib.pbkdf2_hmac(
-                'sha256',
-                self.entry[self.name].encode('utf-8'),
-                self.salt,
-                100000,
-                dklen=12,
-            )
-            self._hash_exists = True
-        finally:
-            return self._hash
-
-    @property
-    def salt(self):
-        """Property to detemine a random salt to use in creation of the hash.
-
-        Returns
-        -------
-        bytes
-            The salt as a bytes object.
-        """
-        self._salt = urandom(32)
-        return self._salt
-
-    @staticmethod
-    def as_string(obj: bytes) -> str:
-        """Static method to return a string from a bytes object.
-
-        Parameters
-        ----------
-        obj : bytes
-            Bytes object to be converted to a string.
-
-        Returns
-        -------
-        str
-            The bytes object as a string.
-        """
-        return bytes.hex(obj)
-
-    @staticmethod
-    def fromhex(obj: str) -> bytes:
-        """Static method to create a bytes object from a string.
-
-        Parameters
-        ----------
-        obj : str
-            String object to be converted to bytes.
-
-        Returns
-        -------
-        bytes
-            The string object as bytes.
-        """
-        return bytes.fromhex(obj)
-
-    @staticmethod
-    def _from_key(config_var) -> Tuple[str, str]:
-        try:
-            header, name = config_var.split('.')
-        except ValueError:
-            header = ''
-            name = config_var
-        return (header, name)
-
-    def __init__(
-        self, config_path: str, config_contents: dict, config_var: str
-    ):
-        """Summary
-        See :class:`~Mask` for parameters.
-        """
-        self.table: str = __header__
-        self.config_path = config_path
-        self.config_contents = config_contents
-        self.config_var = config_var.replace('.', '_')
-        self.header = self._from_key(config_var)[0]
-        self.name = self._from_key(config_var)[1]
-        try:
-            # If value is under a subsection
-            self.entry = self.config_contents[self.table][self.header]
-        except KeyError:
-            # If value is under the main header
-            self.entry = self.config_contents[self.table]
-
-    def _get_database_file(self):
-        self.database = self.config_path
-        self.database += (
-            f'.{self.table}.db'
-            if self.config_path[-1] == '/'
-            else f'/.{self.table}.db'
-        )
-        self.database = path.expanduser(self.database)
-        return self
-
-    def _open_database(self):
-        self.database = Pylite(self.database)
-
-    def _get_table(self):
-        tables = [i[0] for i in self.database.get_tables()]
-        if self.table not in tables:
-            # panaetius.logger.debug(
-            #     'Table not present in the database;'
-            #     f'creating the table {self.table} now'
-            # )
-            self.database.add_table(
-                f'{self.table}',
-                Name='text',
-                Hash='text',
-                Salt='text',
-                Value='text',
-            )
-        else:
-            # panaetius.logger.debug('Table already exists in the database')
-            pass
-        self.table_name = self.table
-
-    def _check_entries(self):
-        var = self.database.get_items(self.table, f'Name="{self.config_var}"')
-        if len(var) == 0:
-            return False
-        else:
-            return True
-
-    def _insert_entries(self):
-        self.database.insert(
-            self.table,
-            self.config_var,
-            self.as_string(self.hash),
-            self.as_string(self.salt),
-            self.entry[self.name],
-        )
-
-    def _update_entries_in_db(self):
-        self.database.remove(self.table, f'Name="{self.config_var}"')
-        self._insert_entries()
-
-    def _run_query(self, query: str):
-        cur = self.database.db.cursor()
-        cur.execute(query)
-        self.database.db.commit()
-        self.result = cur.fetchall()
-        return self
-
-    def _get_all_items(self, where_clause: str = None):
-        if where_clause is not None:
-            self.result = self.database.get_items(self.table, where_clause)
-        else:
-            self.result = self.database.get_items(self.table)
-        return self
-
-    def _process(self):
-        if not self._check_entries():
-            # panaetius.logger.debug('does not exist')
-            self._insert_entries()
-            self._update_entries_in_config()
-            self._get_all_items()
-            # panaetius.logger.debug(f'returning: {self.result[0][3]}')
-            return self.entry[self.name]
-        else:
-            self._get_all_items(f'Name="{self.config_var}"')
-            if self.result[0][1] == self.entry[self.name]:
-                # panaetius.logger.debug('exists and hash matches')
-                # panaetius.logger.debug(f'returning: {self.result[0][3]}')
-                return self.result
-            else:
-                # panaetius.logger.debug('exists and hash doesnt match')
-                # panaetius.logger.debug(
-                #     f'file_hash={self.entry[self.name]}, {self.result[0][1]}'
-                # )
-                self._update_entries_in_db()
-                self._update_entries_in_config()
-                self._get_all_items(f'Name="{self.config_var}"')
-                # panaetius.logger.debug(f'returning: {self.result[0][3]}')
-                return self.entry[self.name]
-
-    def _open_config_file(self) -> io.TextIOWrapper:
-        self.config_path += (
-            '/config.toml' if self.config_path[-1] != '/' else 'config.toml'
-        )
-        c = open(path.expanduser(self.config_path), 'w')
-        return c
-
-    def _update_entries_in_config(self):
-        self.entry.update({self.name: self.as_string(self.hash)})
-        # panaetius.logger.debug(self.config_contents)
-        # panaetius.logger.debug(self.entry)
-        c = self._open_config_file()
-        toml.dump(self.config_contents, c)
-        c.close()
-
-    def get_value(self):
-        """Get the true value from the database if it exists, create if it'
-        ' doesn't exist or update if the hash has changed.
-
-        Returns
-        -------
-        str
-            The result from the database.
-        """
-        # print(f'key in db {self.config_var}')
-        self._get_database_file()
-        self._open_database()
-        self._get_table()
-        self._process()
-        return self.result[0][3]

src/panaetius/header.py

@@ -1,26 +0,0 @@
-import os
-from importlib import util
-
-__path = os.getcwd()
-
-try:
-    __spec = util.spec_from_file_location(
-        '__header__', f'{os.getcwd()}/__header__.py'
-    )
-    __header__ = util.module_from_spec(__spec)
-    __spec.loader.exec_module(__header__)
-    __header__ = __header__.__header__
-except FileNotFoundError:
-    try:
-        venv = os.environ.get('VIRTUAL_ENV').split('/')[-1]
-        __header__ = venv
-    except AttributeError:
-        print(
-            f'Cannot find a __header__.py file in {os.getcwd()} containing the'
-            ' __header__ value of your project name and you are not working'
-            ' from a virtual environment. Either make sure this file '
-            'exists and the value is set or create and work from a virtual '
-            'environment and try again. \n The __header__ value has been '
-            'set to the default of panaetius.'
-        )
-        __header__ = 'panaetius'

src/panaetius/library.py

@@ -1,112 +0,0 @@
-from __future__ import annotations
-import sys
-from typing import Any, TypeVar, Type, TYPE_CHECKING, Union, List
-import ast
-
-
-if TYPE_CHECKING:
-    import logging
-
-
-config_inst_t = TypeVar('config_inst_t', bound='panaetius.config.Config')
-
-
-def export(fn: callable) -> callable:
-    mod = sys.modules[fn.__module__]
-    if hasattr(mod, '__all__'):
-        mod.__all__.append(fn.__name__)
-    else:
-        mod.__all__ = [fn.__name__]
-    return fn
-
-
-def set_config(
-    config_inst: Type[config_inst_t],
-    key: str,
-    default: str = None,
-    cast: Any = None,
-    check: Union[None, List] = None,
-    mask: bool = False,
-) -> None:
-    """Sets the config variable on the instance of a class.
-
-    Parameters
-    ----------
-    config_inst : Type[config_inst_t]
-        Instance of the :class:`~panaetius.config.Config` class.
-    key : str
-        The key referencing the config variable.
-    default : str, optional
-        The default value.
-    mask : bool, optional
-        Boolean to indiciate if a value in the `config.toml` should be masked.
-        If this is set to True then the first time the variable is read from
-        the config file the value will be replaced with a hash. Any time that
-        value is then read the hash will be compared to the one stored and if
-        they match the true value will be returned. This is stored in a sqlite
-        `.db` next to the config file and is hidden by default. If the hash
-        provided doesn't match the default behaviour is to update the `.db`
-        with the new value and hash the value again. If you delete the
-        database file then you will need to set the value again in the
-        `config.toml`.
-    cast : Any, optional
-        The type of the variable.
-    check : Union[None, List], optional
-        Type of object to check against. This is useful if you want to use TOML
-        to define a list, but want to make sure that a string representation
-        of a list will be loaded properly if it set as an environment variable.
-
-        Example:
-
-        *config.toml* has the following attribute set::
-
-            [package.users]
-            auth = ['user1', 'user2']
-
-        If set as an environment variable you can pass this list as a string
-        and set :code:`check=list`::
-
-            Environment variable:
-            PACKAGE_USERS_AUTH = "['user1', 'user2']"
-
-        Usage in code::
-
-            set_config(CONFIG, 'users.auth', check=list)
-    """
-    config_var = key.lower().replace('.', '_')
-    if check is None:
-        setattr(
-            config_inst, config_var, config_inst.get(key, default, cast, mask)
-        )
-    else:
-        if type(config_inst.get(key, default, cast, mask)) is not check:
-            if check is list:
-                var = ast.literal_eval(
-                    config_inst.get(key, default, cast, mask)
-                )
-                setattr(config_inst, config_var, var)
-        else:
-            setattr(
-                config_inst,
-                config_var,
-                config_inst.get(key, default, cast, mask),
-            )
-
-
-# Create function to print cached logged messages and reset
-def process_cached_logs(
-    config_inst: Type[config_inst_t], logger: logging.Logger
-):
-    """Prints the cached messages from :class:`~panaetius.config.Config`
-    and resets the cache.
-
-    Parameters
-    ----------
-    config_inst : Type[config_inst_t]
-        Instance of :class:`~panaetius.config.Config`.
-    logger : logging.Logger
-        Instance of the logger.
-    """
-    for msg in config_inst.deferred_messages:
-        logger.info(msg)
-    config_inst.reset_log()

src/panaetius/logging.py

@@ -1,54 +0,0 @@
-import logging
-from logging.handlers import RotatingFileHandler
-import os
-import sys
-
-import panaetius
-from panaetius import CONFIG as CONFIG
-from panaetius import __header__ as __header__
-from panaetius import set_config as set_config
-
-
-panaetius.set_config(CONFIG, 'logging.path')
-panaetius.set_config(
-    CONFIG,
-    'logging.format',
-    '{\n\t"time": "%(asctime)s",\n\t"file_name": "%(filename)s",'
-    '\n\t"module": "%(module)s",\n\t"function":"%(funcName)s",\n\t'
-    '"line_number": "%(lineno)s",\n\t"logging_level":'
-    '"%(levelname)s",\n\t"message": "%(message)s"\n}',
-    cast=str,
-)
-set_config(CONFIG, 'logging.level', 'INFO')
-
-# Logging Configuration
-logger = logging.getLogger(__header__)
-loghandler_sys = logging.StreamHandler(sys.stdout)
-
-# Checking if log path is set
-if CONFIG.logging_path:
-    CONFIG.logging_path += (
-        f'{__header__}.log'
-        if CONFIG.logging_path[-1] == '/'
-        else f'/{__header__}.log'
-    )
-    # Set default log file options
-    set_config(CONFIG, 'logging.backup_count', 3, int)
-    set_config(CONFIG, 'logging.rotate_bytes', 512000, int)
-
-    # Configure file handler
-    loghandler_file = RotatingFileHandler(
-        os.path.expanduser(CONFIG.logging_path),
-        'a',
-        CONFIG.logging_rotate_bytes,
-        CONFIG.logging_backup_count,
-    )
-
-    # Add to file formatter
-    loghandler_file.setFormatter(logging.Formatter(CONFIG.logging_format))
-    logger.addHandler(loghandler_file)
-
-# Configure and add to stdout formatter
-loghandler_sys.setFormatter(logging.Formatter(CONFIG.logging_format))
-logger.addHandler(loghandler_sys)
-logger.setLevel(CONFIG.logging_level)

tests/__header__.py

@@ -1 +0,0 @@
-__header__ = 'panaetius_test'

tests/conftest.py

@@ -0,0 +1,26 @@
+import pytest
+
+
+@pytest.fixture()
+def header():
+    return "panaetius_testing"
+
+
+@pytest.fixture()
+def testing_config_contents():
+    return {
+        "panaetius_testing": {
+            "some_top_string": "some_top_value",
+            "second": {
+                "some_second_string": "some_second_value",
+                "some_second_int": 1,
+                "some_second_float": 1.0,
+                "some_second_list": ["some", "second", "value"],
+                "some_second_table": {"first": ["some", "first", "value"]},
+                "some_second_table_bools": {"bool": [True, False]},
+                "third": {
+                    "some_third_string": "some_third_value",
+                },
+            },
+        }
+    }

tests/data/without_header/config.yml

@@ -0,0 +1,9 @@
+panaetius_testing:
+  some_top_string: some_top_value
+  second:
+    some_second_string: some_second_value
+    some_second_int: 1
+    some_second_float: 1.0
+    some_second_list: ["some", "second", "value"]
+    some_second_table: { "first": ["some", "first", "value"] }
+    some_second_table_bools: { "bool": [true, false] }

tests/data/without_logging/panaetius_testing/config.toml

@@ -0,0 +1,10 @@
+[panaetius_testing]
+some_top_string = "some_top_value"
+
+[panaetius_testing.second]
+some_second_string = "some_second_value"
+some_second_int = 1
+some_second_float = 1.0
+some_second_list = ["some", "second", "value"]
+some_second_table = { "first" = ["some", "first", "value"] }
+some_second_table_bools = { "bool" = [true, false] }

tests/data/without_logging/panaetius_testing/config.yml

@@ -0,0 +1,11 @@
+panaetius_testing:
+  some_top_string: some_top_value
+  second:
+    some_second_string: some_second_value
+    some_second_int: 1
+    some_second_float: 1.0
+    some_second_list: ["some", "second", "value"]
+    some_second_table: { "first": ["some", "first", "value"] }
+    some_second_table_bools: { "bool": [true, false] }
+    third:
+      some_third_string: some_third_value

tests/scratchpad.py

@@ -0,0 +1,57 @@
+import os
+
+from panaetius import Config, set_config, set_logger, SimpleLogger
+
+from panaetius.logging import AdvancedLogger
+
+if __name__ == "__main__":
+    os.environ["PANAETIUS_TEST_PATH"] = '"/usr/local"'
+    os.environ["PANAETIUS_TEST_BOOL"] = "True"
+    # print(os.environ.get("PANAETIUS_TEST_PATH"))
+    # os.environ[
+    #     "PANAETIUS_TEST_TOML_POINTS"
+    # ] = "[ { x = 1, y = 2, z = 3 }, { x = 7, y = 8, z = 9 }, { x = 2, y = 4, z = 8 }]"
+
+    os.environ["PANAETIUS_TEST_NOC_PATH"] = '"/usr/locals"'
+    os.environ["PANAETIUS_TEST_NOC_FLOAT"] = "2.0"
+    os.environ["PANAETIUS_TEST_NOC_BOOL"] = "True"
+    os.environ["PANAETIUS_TEST_NOC_EMBEDDED_PATH"] = '"/usr/local"'
+    os.environ["PANAETIUS_TEST_NOC_EMBEDDED_FLOAT"] = "2.0"
+    os.environ["PANAETIUS_TEST_NOC_EMBEDDED_BOOL"] = "True"
+
+    c = Config("panaetius_test")
+    # c = Config("panaetius_test_noc")
+
+    set_config(c, key="toml.points")
+    set_config(c, key="path", default="some path")
+    set_config(c, key="top", default="some top")
+    set_config(c, key="logging.path")
+    set_config(c, key="nonexistent.item", default="some nonexistent item")
+    set_config(c, key="nonexistent.item")
+    set_config(c, key="toml.points_config")
+    set_config(c, key="float")
+    set_config(c, key="float_str", default="2.0")
+    set_config(c, key="bool")
+    set_config(c, key="noexistbool", default=False)
+    set_config(c, key="middle.middle")
+
+    # set_config(c, key="path")
+    # set_config(c, key="float")
+    # set_config(c, key="bool")
+    # set_config(c, key="noexiststr", default="2.0")
+    # set_config(c, key="noexistfloat", default=2.0)
+    # set_config(c, key="noexistbool", default=False)
+
+    set_config(c, key="embedded.path")
+    set_config(c, key="embedded.float")
+    set_config(c, key="embedded.bool")
+    set_config(c, key="embedded.noexiststr", default="2.0")
+    set_config(c, key="embedded.noexistfloat", default=2.0)
+    set_config(c, key="embedded.noexistbool", default=False)
+
+    # logger = set_logger(c, SimpleLogger())
+    logger = set_logger(c, SimpleLogger(logging_level="DEBUG"))
+    logger.info("some logging message")
+    logger.debug("debugging message")
+    # for i in dir(c):
+    #     logger.debug(i + ": " + str(getattr(c, i)) + " - " + str(type(getattr(c, i))))

tests/test.py

@@ -1,33 +0,0 @@
-import panaetius
-
-# from panaetius import CONFIG as CONFIG
-# from panaetius import logger as logger
-
-print(panaetius.__header__)
-
-panaetius.set_config(panaetius.CONFIG, 'logging.level')
-
-# print(panaetius.CONFIG.logging_format)
-print(panaetius.CONFIG.logging_path)
-print(panaetius.config_inst.CONFIG_PATH)
-
-# panaetius.logger.info('test event')
-
-
-panaetius.logger.info('setting foo.bar value')
-panaetius.set_config(panaetius.CONFIG, 'foo.bar', mask=True)
-
-panaetius.logger.info(f'foo.bar set to {panaetius.CONFIG.foo_bar}')
-
-# print((panaetius.CONFIG.path))
-# print(panaetius.CONFIG.logging_level)
-
-panaetius.set_config(panaetius.CONFIG, 'test', mask=True)
-panaetius.logger.info(f'test_root={panaetius.CONFIG.test}')
-
-print(panaetius.CONFIG.config_file)
-
-# for i in panaetius.CONFIG.deferred_messages:
-#     panaetius.logger.debug(i)
-
-panaetius.logger.info('some logging message')

tests/test_config.py

@@ -0,0 +1,268 @@
+import os
+import pathlib
+from uuid import uuid4
+
+import pytest
+
+import panaetius
+from panaetius.exceptions import InvalidPythonException, KeyErrorTooDeepException
+
+# test config paths
+
+
+def test_default_config_path_set(header):
+    # act
+    config = panaetius.Config(header)
+
+    # assert
+    assert str(config.config_path) == str(pathlib.Path.home() / ".config")
+
+
+def test_user_config_path_set(header, shared_datadir):
+    # arrange
+    config_path = str(shared_datadir / "without_logging")
+
+    # act
+    config = panaetius.Config(header, config_path)
+
+    # assert
+    assert str(config.config_path) == config_path
+
+
+def test_user_config_path_without_header_dir_set(header, shared_datadir):
+    # arrange
+    config_path = str(shared_datadir / "without_header")
+
+    # act
+    config = panaetius.Config(header, config_path, skip_header_init=True)
+
+    # assert
+    assert str(config.config_path) == config_path
+
+
+# test config files
+
+
+def test_config_file_exists(header, shared_datadir):
+    # arrange
+    config_path = str(shared_datadir / "without_logging")
+
+    # act
+    config = panaetius.Config(header, config_path)
+    _ = config.config
+
+    # assert
+    assert config._missing_config is False
+
+
+def test_config_file_without_header_dir_exists(header, shared_datadir):
+    # arrange
+    config_path = str(shared_datadir / "without_header")
+
+    # act
+    config = panaetius.Config(header, config_path, skip_header_init=True)
+    _ = config.config
+
+    # assert
+    assert config._missing_config is False
+
+
+def test_config_file_contents_read_success(header, shared_datadir, testing_config_contents):
+    # arrange
+    config_path = str(shared_datadir / "without_logging")
+
+    # act
+    config = panaetius.Config(header, config_path)
+    config_contents = config.config
+
+    # assert
+    assert config_contents == testing_config_contents
+
+
+@pytest.mark.parametrize(
+    "set_config_key,get_config_key,expected_value",
+    [
+        ("some_top_string", "some_top_string", "some_top_value"),
+        ("second.some_second_string", "second_some_second_string", "some_second_value"),
+        (
+            "second.some_second_list",
+            "second_some_second_list",
+            ["some", "second", "value"],
+        ),
+        (
+            "second.some_second_table",
+            "second_some_second_table",
+            {"first": ["some", "first", "value"]},
+        ),
+        (
+            "second.some_second_table_bools",
+            "second_some_second_table_bools",
+            {"bool": [True, False]},
+        ),
+        ("second.third.some_third_string", "second_third_some_third_string", "some_third_value"),
+    ],
+)
+def test_get_value_from_key(set_config_key, get_config_key, expected_value, header, shared_datadir):
+    """
+    Test the following:
+
+    - keys are read from top level key
+    - keys are read from two level key
+    - inline arrays are read correctly
+    - inline tables are read correctly
+    - inline tables & arrays read bools correctly
+    """
+    # arrange
+    config_path = str(shared_datadir / "without_logging")
+    config = panaetius.Config(header, config_path)
+    panaetius.set_config(config, set_config_key)
+
+    # act
+    config_value = getattr(config, get_config_key)
+
+    # assert
+    assert config_value == expected_value
+
+
+def test_get_value_environment_var_override(header, shared_datadir):
+    # arrange
+    os.environ[f"{header.upper()}_SOME_TOP_STRING"] = "some_overridden_value"
+    config_path = str(shared_datadir / "without_logging")
+    config = panaetius.Config(header, config_path)
+    panaetius.set_config(config, "some_top_string")
+
+    # act
+    config_value = getattr(config, "some_top_string")
+
+    # assert
+    assert config_value == "some_overridden_value"
+
+    # cleanup
+    del os.environ[f"{header.upper()}_SOME_TOP_STRING"]
+
+
+def test_key_level_too_deep(header, shared_datadir):
+    # arrange
+    config_path = str(shared_datadir / "without_logging")
+    config = panaetius.Config(header, config_path)
+    key = "a.key.too.deep"
+
+    # act
+    with pytest.raises(KeyErrorTooDeepException) as key_error_too_deep:
+        panaetius.set_config(config, key)
+
+    # assert
+    assert str(key_error_too_deep.value) == f"Your key of {key} can only be 3 levels deep maximum."
+
+
+def test_get_value_missing_key_from_default(header, shared_datadir):
+    # arrange
+    config_path = str(shared_datadir / "without_logging")
+    config = panaetius.Config(header, config_path)
+    panaetius.set_config(
+        config,
+        "missing.key_from_default",
+        default=["some", "default", "value", 1.0, True],
+    )
+
+    # act
+    default_value = getattr(config, "missing_key_from_default")
+
+    # assert
+    assert default_value == ["some", "default", "value", 1.0, True]
+
+
+def test_get_value_missing_key_from_env(header, shared_datadir):
+    # arrange
+    os.environ[f"{header.upper()}_MISSING_KEY"] = "some missing key"
+
+    config_path = str(shared_datadir / "without_logging")
+    config = panaetius.Config(header, config_path)
+    panaetius.set_config(config, "missing_key")
+
+    # act
+    value_from_key = getattr(config, "missing_key")
+
+    # assert
+    assert value_from_key == "some missing key"
+
+    # cleanup
+    del os.environ[f"{header.upper()}_MISSING_KEY"]
+
+
+# test env vars
+
+
+def test_config_file_does_not_exist(header, shared_datadir):
+    # arrange
+    config_path = str(shared_datadir / "nonexistent_folder")
+
+    # act
+    config = panaetius.Config(header, config_path)
+    config_contents = config.config
+
+    # assert
+    assert config._missing_config is True
+    assert config_contents == {}
+
+
+def test_missing_config_read_from_default(header, shared_datadir):
+    # arrange
+    config_path = str(shared_datadir / "nonexistent_folder")
+
+    # act
+    config = panaetius.Config(header, config_path)
+    panaetius.set_config(config, "missing.key_read_from_default", default=True)
+
+    # assert
+    assert getattr(config, "missing_key_read_from_default") is True
+
+
+@pytest.mark.parametrize(
+    "env_value,expected_value",
+    [
+        ("a missing string", "a missing string"),
+        ("1", 1),
+        ("1.0", 1.0),
+        ("True", True),
+        (
+            '["an", "array", "of", "items", 1, True]',
+            ["an", "array", "of", "items", 1, True],
+        ),
+        (
+            '{"an": "array", "of": "items", "1": True}',
+            {"an": "array", "of": "items", "1": True},
+        ),
+    ],
+)
+def test_missing_config_read_from_env_var(env_value, expected_value, header, shared_datadir):
+    # arrange
+    config_path = str(shared_datadir / str(uuid4()))
+    os.environ[f"{header.upper()}_MISSING_KEY_READ_FROM_ENV_VAR"] = env_value
+
+    # act
+    config = panaetius.Config(header, config_path)
+    panaetius.set_config(config, "missing.key_read_from_env_var")
+
+    # assert
+    assert getattr(config, "missing_key_read_from_env_var") == expected_value
+
+    # cleanup
+    del os.environ[f"{header.upper()}_MISSING_KEY_READ_FROM_ENV_VAR"]
+
+
+@pytest.mark.skip(reason="No longer needed as strings are loaded without quotes")
+def test_missing_config_read_from_env_var_invalid_python(header):
+    # arrange
+    os.environ[f"{header.upper()}_INVALID_PYTHON"] = "a string without quotes"
+    config = panaetius.Config(header)
+
+    # act
+    with pytest.raises(InvalidPythonException) as invalid_python_exception:
+        panaetius.set_config(config, "invalid_python")
+
+    # assert
+    assert str(invalid_python_exception.value) == "a string without quotes is not valid Python."
+
+    # cleanup
+    del os.environ[f"{header.upper()}_INVALID_PYTHON"]

tests/test_library.py

@@ -0,0 +1,13 @@
+import panaetius
+
+
+def test_set_config(header, shared_datadir):
+    # arrange
+    config_path = str(shared_datadir / "without_logging")
+
+    # act
+    config = panaetius.Config(header, config_path)
+    panaetius.set_config(config, "some_top_string")
+
+    # assert
+    assert getattr(config, "some_top_string") == "some_top_value"

tests/test_logging.py

@@ -0,0 +1,37 @@
+import logging
+from uuid import uuid4
+
+import pytest
+
+from panaetius import set_logger, SimpleLogger, Config, set_config
+from panaetius.exceptions import LoggingDirectoryDoesNotExistException
+
+
+def test_logging_directory_does_not_exist(header, shared_datadir):
+    # arrange
+    config = Config(header)
+    logging_path = str(shared_datadir / str(uuid4()))
+    set_config(config, "logging.path", default=str(logging_path))
+
+    # act
+    with pytest.raises(LoggingDirectoryDoesNotExistException) as logging_exception:
+        _ = set_logger(config, SimpleLogger())
+
+    # assert
+    assert str(logging_exception.value) == ""
+
+
+# TODO: change this test so it asserts the dir exists
+def test_logging_directory_does_exist(header, shared_datadir):
+    # arrange
+    config = Config(header)
+    logging_path = str(shared_datadir / "without_logging")
+    set_config(config, "logging.path", default=str(logging_path))
+
+    # act
+    logger = set_logger(config, SimpleLogger())
+
+    # assert
+    assert isinstance(logger, logging.Logger)
+
+# TODO: add tests to check that SimpleLogger, AdvancedLogger, CustomLogger work as intended

tests/test_panaetius.py

@@ -1,5 +0,0 @@
-from panaetius import __version__
-
-
-def test_version():
-    assert __version__ == '0.1.0'

tests/test_utilities/test_squasher.py

@@ -0,0 +1,119 @@
+import pytest
+
+from panaetius import utilities
+
+
+def test_squashed_data(squashed_data, squashed_data_result):
+    # act
+    squashed_data_pre_squashed = utilities.squasher.Squash(squashed_data).as_dict
+
+    # assert
+    assert squashed_data_pre_squashed == squashed_data_result
+
+
+@pytest.fixture
+def squashed_data():
+    return {
+        "destination_addresses": [
+            "Washington, DC, USA",
+            "Philadelphia, PA, USA",
+            "Santa Barbara, CA, USA",
+            "Miami, FL, USA",
+            "Austin, TX, USA",
+            "Napa County, CA, USA",
+        ],
+        "origin_addresses": ["New York, NY, USA"],
+        "rows": [
+            {
+                "elements": [
+                    {
+                        "distance": {"text": "227 mi", "value": 365468},
+                        "duration": {
+                            "text": "3 hours 54 mins",
+                            "value": 14064,
+                        },
+                        "status": "OK",
+                    },
+                    {
+                        "distance": {"text": "94.6 mi", "value": 152193},
+                        "duration": {"text": "1 hour 44 mins", "value": 6227},
+                        "status": "OK",
+                    },
+                    {
+                        "distance": {"text": "2,878 mi", "value": 4632197},
+                        "duration": {
+                            "text": "1 day 18 hours",
+                            "value": 151772,
+                        },
+                        "status": "OK",
+                    },
+                    {
+                        "distance": {"text": "1,286 mi", "value": 2069031},
+                        "duration": {
+                            "text": "18 hours 43 mins",
+                            "value": 67405,
+                        },
+                        "status": "OK",
+                    },
+                    {
+                        "distance": {"text": "1,742 mi", "value": 2802972},
+                        "duration": {"text": "1 day 2 hours", "value": 93070},
+                        "status": "OK",
+                    },
+                    {
+                        "distance": {"text": "2,871 mi", "value": 4620514},
+                        "duration": {
+                            "text": "1 day 18 hours",
+                            "value": 152913,
+                        },
+                        "status": "OK",
+                    },
+                ]
+            }
+        ],
+        "status": "OK",
+    }
+
+
+@pytest.fixture
+def squashed_data_result():
+    return {
+        "destination_addresses_0": "Washington, DC, USA",
+        "destination_addresses_1": "Philadelphia, PA, USA",
+        "destination_addresses_2": "Santa Barbara, CA, USA",
+        "destination_addresses_3": "Miami, FL, USA",
+        "destination_addresses_4": "Austin, TX, USA",
+        "destination_addresses_5": "Napa County, CA, USA",
+        "origin_addresses_0": "New York, NY, USA",
+        "rows_0_elements_0_distance_text": "227 mi",
+        "rows_0_elements_0_distance_value": 365468,
+        "rows_0_elements_0_duration_text": "3 hours 54 mins",
+        "rows_0_elements_0_duration_value": 14064,
+        "rows_0_elements_0_status": "OK",
+        "rows_0_elements_1_distance_text": "94.6 mi",
+        "rows_0_elements_1_distance_value": 152193,
+        "rows_0_elements_1_duration_text": "1 hour 44 mins",
+        "rows_0_elements_1_duration_value": 6227,
+        "rows_0_elements_1_status": "OK",
+        "rows_0_elements_2_distance_text": "2,878 mi",
+        "rows_0_elements_2_distance_value": 4632197,
+        "rows_0_elements_2_duration_text": "1 day 18 hours",
+        "rows_0_elements_2_duration_value": 151772,
+        "rows_0_elements_2_status": "OK",
+        "rows_0_elements_3_distance_text": "1,286 mi",
+        "rows_0_elements_3_distance_value": 2069031,
+        "rows_0_elements_3_duration_text": "18 hours 43 mins",
+        "rows_0_elements_3_duration_value": 67405,
+        "rows_0_elements_3_status": "OK",
+        "rows_0_elements_4_distance_text": "1,742 mi",
+        "rows_0_elements_4_distance_value": 2802972,
+        "rows_0_elements_4_duration_text": "1 day 2 hours",
+        "rows_0_elements_4_duration_value": 93070,
+        "rows_0_elements_4_status": "OK",
+        "rows_0_elements_5_distance_text": "2,871 mi",
+        "rows_0_elements_5_distance_value": 4620514,
+        "rows_0_elements_5_duration_text": "1 day 18 hours",
+        "rows_0_elements_5_duration_value": 152913,
+        "rows_0_elements_5_status": "OK",
+        "status": "OK",
+    }