chore: release v2.3.4

* linting

* adding py.typed

* adding pyyaml types

* workaround #1

* updating latest

* updating dev dependencies

* adding isort mypy safety

* chore: prepare release 2.3.3

.vscode/settings.json

@@ -4,20 +4,5 @@
     "python.linting.enabled": true,
     "python.pythonPath": ".venv/bin/python",
     "restructuredtext.confPath": "${workspaceFolder}/docs/source",
-    "peacock.color": "#307E6A",
-    "workbench.colorCustomizations": {
-        "editorGroup.border": "#3ea389",
-        "panel.border": "#3ea389",
-        "sash.hoverBorder": "#3ea389",
-        "sideBar.border": "#3ea389",
-        "statusBar.background": "#307e6a",
-        "statusBar.foreground": "#e7e7e7",
-        "statusBarItem.hoverBackground": "#3ea389",
-        "statusBarItem.remoteBackground": "#307e6a",
-        "statusBarItem.remoteForeground": "#e7e7e7",
-        "titleBar.activeBackground": "#307e6a",
-        "titleBar.activeForeground": "#e7e7e7",
-        "titleBar.inactiveBackground": "#307e6a99",
-        "titleBar.inactiveForeground": "#e7e7e799"
-    }
+    "peacock.color": "#307E6A"
 }

CHANGELOG.md

@@ -0,0 +1,24 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+<!-- marker -->
+## [v2.3.4](https://github.com/dtomlinson91/panaetius/commits/v2.3.4) - 2022-01-03
+<small>[Compare with 2.3.3](https://github.com/dtomlinson91/panaetius/compare/2.3.3..v2.3.4)</small>
+
+### Documentation
+
+- Update README.md with script boilerplate ([179007](https://github.com/dtomlinson91/panaetius/commit/1790071741207de13330ba75d7bf090106290d72))
+- Update README.md with script quickstart logging ([e6cfde](https://github.com/dtomlinson91/panaetius/commit/e6cfded87dcfc5d2bf62d36bc7b4dbbdeb94b0b8))
+
+### Features
+
+- Add ability to retrieve keys 3 levels deep ([f5ea19](https://github.com/dtomlinson91/panaetius/commit/f5ea19e7d2f977244594b378c6b7633f02f6048a))
+
+### Testing
+
+- Add tests for f5ea19e ([156af4](https://github.com/dtomlinson91/panaetius/commit/156af4685510bac97a850b83d63f8337635db199))
+
+### Build
+
+- Update dependencies ([9bc89f](https://github.com/dtomlinson91/panaetius/commit/9bc89fd2ce9ddf8dcd6a3ca84ef9b72ee183efd3))

README.md

@@ -0,0 +1,119 @@
+# Panaetius
+
+This package provides:
+
+- Functionality to read user variables from a `config.yml` or environment variables.
+- A convenient default logging formatter printing `json` that can save to disk and rotate.
+- Utility functions.
+
+## Config
+
+### options
+
+#### skip_header_init
+
+If `skip_header_init=True` then the `config_path` will not use the `header_variable` as the
+sub-directory in the `config_path`.
+
+E.g
+
+`CONFIG = panaetius.Config("tembo", "~/tembo/.config", skip_header_init=True)`
+
+Will look in `~/tembo/config/config.yml`.
+
+If `skip_header_init=False` then would look in `~/tembo/config/tembo/config.yml`.
+
+### Module
+
+Convenient to place in a package/sub-package `__init__.py`.
+
+See Tembo for an example: <https://github.com/tembo-pages/tembo-core/blob/main/tembo/cli/__init__.py>
+
+Example snippet to use in a module:
+
+```python
+"""Subpackage that contains the CLI application."""
+
+import os
+from typing import Any
+
+import panaetius
+from panaetius.exceptions import LoggingDirectoryDoesNotExistException
+
+
+if (config_path := os.environ.get("TEMBO_CONFIG")) is not None:
+    CONFIG: Any = panaetius.Config("tembo", config_path, skip_header_init=True)
+else:
+    CONFIG = panaetius.Config(
+        "tembo", "~/tembo/.config", skip_header_init=True
+    )
+
+
+panaetius.set_config(CONFIG, "base_path", "~/tembo")
+panaetius.set_config(CONFIG, "template_path", "~/tembo/.templates")
+panaetius.set_config(CONFIG, "scopes", {})
+panaetius.set_config(CONFIG, "logging.level", "DEBUG")
+panaetius.set_config(CONFIG, "logging.path")
+
+try:
+    logger = panaetius.set_logger(
+        CONFIG, panaetius.SimpleLogger(logging_level=CONFIG.logging_level)
+    )
+except LoggingDirectoryDoesNotExistException:
+    _LOGGING_PATH = CONFIG.logging_path
+    CONFIG.logging_path = ""
+    logger = panaetius.set_logger(
+        CONFIG, panaetius.SimpleLogger(logging_level=CONFIG.logging_level)
+    )
+    logger.warning("Logging directory %s does not exist", _LOGGING_PATH)
+
+```
+
+This means in `./tembo/cli/cli.py` you can
+
+```python
+import tembo.cli
+
+# access the CONFIG instance + variables from the config.yml
+tembo.cli.CONFIG
+```
+
+### Script
+
+Create `./config/config.yml` in the same directory as the script.
+
+In the script initialise a `CONFIG` object:
+
+```python
+import pathlib
+
+import panaetius
+
+CONFIG = panaetius.Config(
+    "teenagers_scraper", str(pathlib.Path(__file__).parents[0] / ".config"), skip_header_init=True
+)
+```
+
+Set variables in the same way as the module above.
+
+#### quickstart logging
+
+```python
+import panaetius
+
+
+def get_logger():
+    logging_dir = pathlib.Path(__file__).parents[0] / "logs"
+    logging_dir.mkdir(parents=True, exist_ok=True)
+
+    CONFIG = panaetius.Config("training_data_into_gcp", skip_header_init=True)
+    panaetius.set_config(CONFIG, "logging.level", "DEBUG")
+    panaetius.set_config(CONFIG, "logging.path", logging_dir)
+    return panaetius.set_logger(CONFIG, panaetius.SimpleLogger(logging_level=CONFIG.logging_level))
+```
+
+## Utility Functions
+
+### Squasher
+
+Squashes a json object or Python dictionary into a single level dictionary.

README.rst

@@ -1,62 +0,0 @@
-Author
-=======
-
-Daniel Tomlinson (dtomlinson@panaetius.co.uk)
-
-Requires
-=========
-
-`>= python3.7`
-
-Python requirements
-====================
-
-- toml = "^0.10.0"
-- pylite = "^0.1.0"
-
-Documentation
-==============
-
-Read the documentation on `read the docs`_.
-
-.. _read the docs: https://panaetius.readthedocs.io/en/latest/introduction.html
-
-Installation
-==============
-
-You can install ``panaetius`` the following ways:
-
-Python
--------
-
-.. Attention:: You should install in a python virtual environment
-
-From pypi using pip
-~~~~~~~~~~~~~~~~~~~~
-
-.. code-block:: bash
-
-    pip install panaetius
-
-From local wheel
-~~~~~~~~~~~~~~~~~
-
-Download the latest verion from the `releases`_ page.
-
-.. _releases: https://github.com/dtomlinson91/panaetius/releases
-
-Install with pip:
-
-.. code-block:: bash
-
-    pip install -U panaetius-1.0.2-py3-none-any.whl
-
-
-From source
-~~~~~~~~~~~~
-
-Clone the repo and install using ``setup.py``:
-
-.. code-block:: bash
-
-    python setup.py

cliff.toml

@@ -0,0 +1,57 @@
+# configuration file for git-cliff (0.1.0)
+
+[changelog]
+# changelog header
+header = """
+# Changelog
+All notable changes to this project will be documented in this file.\n
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).\n
+"""
+# template for the changelog body
+# https://tera.netlify.app/docs/#introduction
+body = """
+{% if version %}\
+    ## [{{ version }}](https://github.com/dtomlinson91/panaetius/commits/{{ version }}) - {{ timestamp | date(format="%Y-%m-%d") }}
+    {% if previous.version %}\
+    <small>[Compare with {{ previous.version }}](https://github.com/dtomlinson91/panaetius/compare/{{ previous.version }}...{{ version }})</small>
+    {% endif %}\
+{% else %}\
+    ## [unreleased]
+{% endif %}\
+{% for group, commits in commits | group_by(attribute="group") %}
+    ### {{ group | upper_first }}
+    {% for commit in commits %}
+        - {{ commit.message | upper_first }} ([{{ commit.id | truncate(length=6, end="") }}](https://github.com/dtomlinson91/panaetius/commit/{{ commit.id }}))\
+    {% endfor %}
+{% endfor %}\n
+"""
+# remove the leading and trailing whitespaces from the template
+trim = true
+# changelog footer
+footer = """
+"""
+
+[git]
+# allow only conventional commits
+# https://www.conventionalcommits.org
+conventional_commits = true
+# regex for parsing and grouping commits
+commit_parsers = [
+    { message = "^feat", group = "Features"},
+    { message = "^fix", group = "Bug Fixes"},
+    { message = "^doc", group = "Documentation"},
+    { message = "^perf", group = "Performance"},
+    { message = "^refactor", group = "Refactor"},
+    { message = "^style", group = "Styling"},
+    { message = "^test", group = "Testing"},
+    { message = "^chore\\(release\\): prepare for", skip = true},
+    { message = "^chore", group = "Miscellaneous Tasks", skip = true},
+    { body = ".*security", group = "Security"},
+]
+# filter out the commits that are not matched by commit parsers
+filter_commits = false
+# glob pattern for matching git tags
+tag_pattern = "v[0-9]*"
+# regex for skipping tags
+skip_tags = "v0.1.0-beta.1"

docs/Makefile

@@ -1,20 +0,0 @@
-# Minimal makefile for Sphinx documentation
-#
-
-# You can set these variables from the command line, and also
-# from the environment for the first two.
-SPHINXOPTS    ?=
-SPHINXBUILD   ?= sphinx-build
-SOURCEDIR     = source
-BUILDDIR      = build
-
-# Put it first so that "make" without argument is like "make help".
-help:
-	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
-
-.PHONY: help Makefile
-
-# Catch-all target: route all unknown targets to Sphinx using the new
-# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
-%: Makefile
-	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/make.bat

@@ -1,35 +0,0 @@
-@ECHO OFF
-
-pushd %~dp0
-
-REM Command file for Sphinx documentation
-
-if "%SPHINXBUILD%" == "" (
-	set SPHINXBUILD=sphinx-build
-)
-set SOURCEDIR=source
-set BUILDDIR=build
-
-if "%1" == "" goto help
-
-%SPHINXBUILD% >NUL 2>NUL
-if errorlevel 9009 (
-	echo.
-	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
-	echo.installed, then set the SPHINXBUILD environment variable to point
-	echo.to the full path of the 'sphinx-build' executable. Alternatively you
-	echo.may add the Sphinx directory to PATH.
-	echo.
-	echo.If you don't have Sphinx installed, grab it from
-	echo.http://sphinx-doc.org/
-	exit /b 1
-)
-
-%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
-goto end
-
-:help
-%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
-
-:end
-popd

docs/source/_static/custom.css

@@ -1,87 +0,0 @@
-@import url("css/theme.css");
-
-.modname {
-    font-size: 0.8em;
-    opacity: 0.4;
-}
-.modname::before {
-    content: '- ';
-}
-.title {
-    font-weight: bold;
-    font-size: 1.2em;
-    background-color: #eee;
-    display: block;
-    padding: 1px 5px;
-    border-left: 2px solid #ddd;
-}
-
-/*colour of the text in the toc*/
-.wy-menu-vertical header, .wy-menu-vertical p.caption{
-    color: #b26d46;;
-}
-
-/*colour of the top left header*/
-.wy-side-nav-search{
-    background-color: #31465a;
-}
-
-/*colours of the class definitions*/
-.rst-content dl:not(.docutils) dt{
-    background: #e2d7d1;
-    color: #0b2852;
-    border-top: solid 3px #31465a;
-}
-
-/*colour of the link in the class defintions*/
-.rst-content .viewcode-link, .rst-content .viewcode-back{
-    color: #4b674a;
-}
-
-/*colour of the function definitions*/
-.rst-content dl:not(.docutils) dl dt{
-    border-left: solid 3px #31465a;
-    background: #e2d7d1;
-    color: #0b2852;
-}
-
-/*colour of the link in the function definitions*/
-.rst-content .viewcode-link, .rst-content .viewcode-back{
-    color: #4b674a;
-}
-
-/*edit the width of the body*/
-.wy-nav-content{
-    max-width: 1200px;
-}
-
-/*code example blocks*/
-.rst-content pre.literal-block, .rst-content div[class^="highlight"] pre, .rst-content .linenodiv pre{
-    background: #b4bec8;
-    color: black;
-    /*border-style: solid;*/
-    /*border-width: thin;*/
-}
-
-/*colour of inline code blocks using ``*/
-.rst-content tt.literal, .rst-content tt.literal, .rst-content code.literal{
-    color: #b26d46;
-}
-
-/* Change code blocks font and bump up font size slightly (normally 12px)*/
-.rst-content pre.literal-block, .rst-content div[class^="highlight"] pre, .rst-content .linenodiv pre {
-    font-family: 'Inconsolata', monospace !important;
-    font-size: 14px !important;
-    white-space: pre-wrap;
-}
-
-/* Change code descriptions and literal blocks (inline code via ``) to match the normal font size being used in the sphinx_RTD_theme text  (normally 14px)*/
-.rst-content dl:not(.docutils) tt.descname,.rst-content dl:not(.docutils) tt.descclassname,.rst-content dl:not(.docutils) tt.descname,.rst-content dl:not(.docutils) code.descname,.rst-content dl:not(.docutils) tt.descclassname,.rst-content dl:not(.docutils) code.descclassname, code.docutils {
-    font-family: 'Inconsolata', monospace !important;
-    font-size: 14px !important;
-}
-
-/*variables text*/
-dl.class > dd > table.docutils.field-list tbody tr.field-odd.field th.field-name::before{
-    content: '(Class Attributes) ';
-}

docs/source/changelog.rst

@@ -1,17 +0,0 @@
-Version history
-================
-
-1.1
-----
-
-- Adding overwrite to ``__header__`` functionality. See the Configuration documentation page on how to configure.
-
-1.0.2
-------
-
-- Minor fixes and documentation updates.
-
-1.0
---------
-
-- Initial release.

docs/source/conf.py

@@ -1,95 +0,0 @@
-# Configuration file for the Sphinx documentation builder.
-#
-# This file only contains a selection of the most common options. For a full
-# list see the documentation:
-# https://www.sphinx-doc.org/en/master/usage/configuration.html
-
-# -- Path setup --------------------------------------------------------------
-
-# If extensions (or modules to document with autodoc) are in another directory,
-# add these directories to sys.path here. If the directory is relative to the
-# documentation root, use os.path.abspath to make it absolute, like shown here.
-#
-# import os
-# import sys
-# sys.path.insert(0, os.path.abspath('.'))
-import panaetius
-from panaetius.__version__ import __version__ as version
-import sphinx_rtd_theme
-
-# -- Project information -----------------------------------------------------
-
-project = 'panaetius'
-copyright = '2019, Daniel Tomlinson'
-author = 'Daniel Tomlinson'
-
-# The full version, including alpha/beta/rc tags
-release = version
-
-
-# -- General configuration ---------------------------------------------------
-
-# Add any Sphinx extension module names here, as strings. They can be
-# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
-# ones.
-extensions = [
-    'sphinx.ext.autodoc',
-    'sphinx.ext.viewcode',
-    'sphinx.ext.napoleon',
-    'sphinx.ext.todo',
-]
-
-# -- Napoleon Settings -----------------------------------------------------
-napoleon_google_docstring = False
-napoleon_numpy_docstring = True
-napoleon_include_init_with_doc = True
-napoleon_include_private_with_doc = False
-napoleon_include_special_with_doc = False
-napoleon_use_admonition_for_examples = False
-napoleon_use_admonition_for_notes = False
-napoleon_use_admonition_for_references = False
-napoleon_use_ivar = True
-napoleon_use_param = True
-napoleon_use_rtype = True
-napoleon_use_keyword = True
-autodoc_member_order = 'bysource'
-
-# Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
-
-# The master toctree document.
-master_doc = 'index'
-
-# List of patterns, relative to source directory, that match files and
-# directories to ignore when looking for source files.
-# This pattern also affects html_static_path and html_extra_path.
-exclude_patterns = []
-
-
-# -- Options for HTML output -------------------------------------------------
-
-# The theme to use for HTML and HTML Help pages.  See the documentation for
-# a list of builtin themes.
-#
-html_theme = "sphinx_rtd_theme"
-
-# Add any paths that contain custom static files (such as style sheets) here,
-# relative to this directory. They are copied after the builtin static files,
-# so a file named "default.css" will overwrite the builtin "default.css".
-
-html_theme = "sphinx_rtd_theme"
-html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
-html_static_path = ['_static']
-html_context = {'css_files': ['_static/custom.css']}
-html_theme_options = {
-    'collapse_navigation': True,
-    'display_version': True,
-    'prev_next_buttons_location': 'both',
-    #'navigation_depth': 3,
-}
-
-# The name of the Pygments (syntax highlighting) style to use.
-pygments_style = 'sphinx'
-
-# Enable todo
-todo_include_todos = True

docs/source/configuration.rst

@@ -1,68 +0,0 @@
-Configuration
-=============
-
-panaetius is fairly easy to configure. There are just a couple of options to be aware of.
-
-Manual configuration of ``Config`` instance
---------------------------------------------
-
-Configuring with a ``__header__.py`` is deprecated. Manually set this value.
-
-Use the following snippet to configure (in ``__init__.py``):
-
-.. code-block:: python
-
-    import panaetius
-    from panaetius.config import Config
-
-    CONFIG = Config(path="~/.config/island-code-extractor", header="island-code-extractor")
-
-    panaetius.set_config(CONFIG, "reddit.secret")
-
-Access this in your code by importing the ``CONFIG`` instance from your module:
-
-.. code-block:: python
-
-    from island_code_extractor import CONFIG
-    from island_code_extractor import panaetius
-
-    CONFIG.reddit_output_path
-    panaetius.logger.info("Using logger")
-
-__header__.py
--------------
-
-You should set a ``__header__.py`` next to your script or module.
-
-This ``__header__.py`` should contain a ``__header__`` variable that sets the name of your project/script.
-
-E.g a ``__header__.py`` for the module ``plex_posters`` would look like:
-
-.. code-block:: python
-
-    __header__ = 'plex_posters' 
-
-Your config file can then be created at ``~/.config/__header__/config.toml``. 
-
-Your environment variables can be created with:
-
-.. code-block:: bash
-
-    HEADER_FOO = "bar"
-    HEADER_SUBSECTION_FOO = "bar"
-
-The headers of the toml file would look like:
-
-.. code-block:: toml
-
-    [__header__]
-    foo = bar
-
-    [__header__.subsection]
-    foo = bar
-
-If you are writing a script, simply place this ``__header__.py`` along side your script. Panaetius will pick this up when the script is ran.
-
-If you are writing a module, you can either place the ``__header__.py`` alongside the script that uses your module. If this is not possible, panaetius will set the default ``__header__`` variable to the name of the virtualenv that the script is activated from.
-
-If neither of the above aren't possible (say your script is running in a lambda on AWS), then ``__header__`` will be set to the default of ``panaetius``.

docs/source/global.rst

@@ -1,5 +0,0 @@
-.. role:: modname
-   :class: modname
-
-.. role:: title
-   :class: title

docs/source/index.rst

@@ -1,3 +0,0 @@
-Table of Contents
-=================
-.. include:: toc.rst

docs/source/introduction.rst

@@ -1,154 +0,0 @@
-Introduction
-=============
-
-.. image:: https://img.shields.io/readthedocs/panaetius?style=for-the-badge   :target: https://panaetius.readthedocs.io/en/latest/?badge=latest
-   :alt: Documentation Status
-
-.. image:: https://img.shields.io/github/v/tag/dtomlinson91/panaetius?style=for-the-badge   :alt: GitHub tag (latest by date)
-
-.. image:: https://img.shields.io/github/commit-activity/m/dtomlinson91/panaetius?style=for-the-badge   :alt: GitHub commit activity
-
-.. image:: https://img.shields.io/github/issues/dtomlinson91/panaetius?style=for-the-badge   :alt: GitHub issues
-
-.. image:: https://img.shields.io/github/license/dtomlinson91/panaetius?style=for-the-badge   :alt: GitHubtbc
-
-Python module to gracefully handle a .config file/environment variables for scripts, with built in masking for sensitive options. Provides a Splunk friendly formatted logger instance.
-
-Usage
-------
-
-Setting a config file
-~~~~~~~~~~~~~~~~~~~~~~
-
-The main functionality of ``panaetius`` is using a config file to store variables.
-
-Your ``config.toml`` can be created and found in ``~/.config/__header__/config.toml`` where ``__header__`` is equal to the variable configured/set. `See how to configure`_ this variable in the configuration section of panaetius.
-
-.. _See how to configure: https://panaetius.readthedocs.io/en/latest/configuration.html#header-py
-
-Setting values in a config.toml/environment variables
-#######################################################
-
-A ``config.toml`` can be created in the default folder for the module. In this example this would be found in ``~/.config/example_module/config.toml``. 
-
-An example ``config.toml`` could look like:
-
-.. code-block:: toml
-
-    [example_module]
-    test = "a6cbf36649b029f3618a0cc1"
-
-    [example_module.logging]
-    path = "~/.config/example_module"
-    level = "DEBUG"
-
-    [example_module.foo]
-    bar = "6b3b96815218960ceaf7cceb"
-
-These are equivalent to the environment variables:
-
-.. code-block:: bash
-
-    EXAMPLE_MODULE_TEST
-    EXAMPLE_MODULE_LOGGING_PATH
-    EXAMPLE_MODULE_LOGGING_LEVEL
-    EXAMPLE_MODULE_FOO_BAR
-
-
-.. Attention:: 
-    Environment variables take precedent over the ``config.toml``. If both are set then the environment variable will be used.
-
-You can overwrite the ``config.toml`` location by setting the environment variable:
-
-.. code-block:: bash
-
-    DEFAULT_CONFIG_PATH = "~/path/to/config"
-
-
-Setting values in your code
-############################
-
-Values in a ``config.toml`` or from an environment variable need to be set in your work in order for you to use them. You can do this easily by
-
-- importing panaetius.
-- using the :func:`~panaetius.library.set_config` function.
-  
-E.g your script could contain:
-
-.. code-block:: python
-
-    import panaetius
-    panaetius.set_config(panaetius.CONFIG, 'logging.path')
-
-.. Note::
-
-    The ``key`` attribute in :func:`~panaetius.library.set_config` is specified as a string, with the hirearchy in the config file split with a ``.``
-
-.. Important::
-
-    The default value for a variable defined using :func:`~panaetius.library.set_config` is ``None``. See the documentation of this function to see all the options available.
-
-
-Accessing values
-#################
-
-You can then access the result of this variable later in your code:
-
-.. code-block:: python
-
-    panaetius.CONFIG.logging_path
-
-
-Logging
-~~~~~~~~
-
-In order to save to disk, you need to specify a path for the log file in the config file/environment variable. There is no need to register this with :func:`~panaetius.library.set_config` as ``panaetius`` will do this automatically.
-
-There are other options available for you to configure a logger. These are (including the default values which can be overwritten):
-
-.. code-block:: toml
-
-    [example_module.logging]
-    backup_count = 3
-    format = "{\n\t"time": "%(asctime)s",\n\t"file_name": "%(filename)s",'
-    '\n\t"module": "%(module)s",\n\t"function":"%(funcName)s",\n\t'
-    '"line_number": "%(lineno)s",\n\t"logging_level":'
-    '"%(levelname)s",\n\t"message": "%(message)s"\n}"
-    level = "INFO" # Level should be in CAPS
-    rotate_bytes = 512000
-
-You can use the logger in your code by:
-
-.. code-block:: python
-
-    panaetius.logger.info('some log message')
-
-which gives an output of:
-
-.. code-block:: json
-
-    {
-        "time": "2020-01-13 23:07:17,913",
-        "file_name": "test.py",
-        "module": "test",
-        "function":"<module>",
-        "line_number": "33",
-        "logging_level":"INFO",
-        "message": "some logging message"
-    }
-
-
-Importing and using the api
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-See `panaetius api page`_ on how to use and import the module.
-
-.. _panaetius api page: https://panaetius.readthedocs.io/en/latest/modules/panaetius.html
-
-
-Configuration
----------------
-
-See `configuration page`_ on how to configure ``panaetius``.
-
-.. _configuration page: https://panaetius.readthedocs.io/en/latest/configuration.html

docs/source/modules/panaetius.config.rst

@@ -1,9 +0,0 @@
-.. include:: ../global.rst
-
-panaetius.config :modname:`panaetius.config`
----------------------------------------------
-
-.. automodule:: panaetius.config
-   :members:
-   :undoc-members:
-   :show-inheritance:

docs/source/modules/panaetius.config_inst.rst

@@ -1,9 +0,0 @@
-.. include:: ../global.rst
-
-panaetius.config_inst :modname:`panaetius.config_inst`
---------------------------------------------------------
-
-.. automodule:: panaetius.config_inst
-   :members:
-   :undoc-members:
-   :show-inheritance:

docs/source/modules/panaetius.db.rst

@@ -1,9 +0,0 @@
-.. include:: ../global.rst
-
-panaetius.db :modname:`panaetius.db`
--------------------------------------
-
-.. automodule:: panaetius.db
-   :members:
-   :undoc-members:
-   :show-inheritance:

docs/source/modules/panaetius.header.rst

@@ -1,9 +0,0 @@
-.. include:: ../global.rst
-
-panaetius.header :modname:`panaetius.header`
----------------------------------------------
-
-.. automodule:: panaetius.header
-   :members:
-   :undoc-members:
-   :show-inheritance:

docs/source/modules/panaetius.library.rst

@@ -1,9 +0,0 @@
-.. include:: ../global.rst
-
-panaetius.library :modname:`panaetius.library`
-------------------------------------------------
-
-.. automodule:: panaetius.library
-   :members:
-   :undoc-members:
-   :show-inheritance:

docs/source/modules/panaetius.logging.rst

@@ -1,9 +0,0 @@
-.. include:: ../global.rst
-
-panaetius.logging :modname:`panaetius.logging`
-----------------------------------------------
-
-.. automodule:: panaetius.logging
-   :members:
-   :undoc-members:
-   :show-inheritance:

docs/source/modules/panaetius.rst

@@ -1,66 +0,0 @@
-.. include:: ../global.rst
-
-*********
-panaetius
-*********
-
-API
-===
-
-The following is availble by importing the module:
-
-.. code-block:: python
-
-    import panaetius
-
-
-panaetius.CONFIG
-----------------
-
-:obj:`panaetius.CONFIG` provides an instance of :class:`panaetius.config.Config`
-
-
-panaetius.set_config()
------------------------
-
-Conveniently provides :func:`panaetius.library.set_config` 
-
-Use in your module/script with:
-
-.. code-block:: python
-
-    panaetius.set_config(panaetius.CONFIG, 'aws.secret_key', str, mask=True)    
-
-
-panaetius.CONFIG.aws_secret_key
--------------------------------
-
-Conveniently provides access to all attributes that have been declared with :func:`panaetius.library.set_config`:
-
-.. code-block:: python
-
-    my_secret_key = panaetius.CONFIG.aws_secret_key
-
-
-panaetius.logger
------------------
-
-:obj:`panaetius.logger` provides a logger instance already formatted with a nice json output.
-
-.. code-block:: python
-
-    panaetius.logger.info('some logging message')
-
-This gives a logger output of:
-
-.. code-block:: json
-
-    {
-        "time": "2020-01-13 23:07:17,913",
-        "file_name": "test.py",
-        "module": "test",
-        "function":"<module>",
-        "line_number": "33",
-        "logging_level":"INFO",
-        "message": "some logging message"
-    }

docs/source/toc.rst

@@ -1,27 +0,0 @@
-.. toctree::
-   :maxdepth: 1
-   :caption: Overview
-   :titlesonly:
-
-   introduction
-   configuration
-   changelog
-
-.. toctree::
-   :maxdepth: 4
-   :caption: Modules
-   :titlesonly:
-
-   modules/panaetius.rst
-
-.. toctree::
-   :maxdepth: 4
-   :caption: Submodules
-   :titlesonly:
-
-   modules/panaetius.config.rst
-   modules/panaetius.config_inst.rst
-   modules/panaetius.db.rst
-   modules/panaetius.header.rst
-   modules/panaetius.library.rst
-   modules/panaetius.logging.rst

duties.py

@@ -0,0 +1,377 @@
+from __future__ import annotations
+
+import importlib
+import os
+import pathlib
+import re
+import shutil
+import sys
+from io import StringIO
+from typing import List, Optional, Pattern
+from urllib.request import urlopen
+
+from duty import duty
+
+PACKAGE_NAME = "panaetius"
+REPO_URL = "https://github.com/dtomlinson91/panaetius"
+
+
+@duty(post=["export"])
+def update_deps(ctx, dry: bool = False):
+    """
+    Update the dependencies using Poetry.
+
+    Args:
+        ctx: The context instance (passed automatically).
+        dry (bool, optional) = If True will update the `poetry.lock` without updating the
+            dependencies themselves. Defaults to False.
+
+    Example:
+        `duty update_deps dry=False`
+    """
+    dry_run = "--dry-run" if dry else ""
+    ctx.run(
+        ["poetry", "update", dry_run],
+        title=f"Updating poetry deps {dry_run}",
+    )
+
+
+@duty
+def test(ctx):
+    """
+    Run tests using pytest.
+
+    Args:
+        ctx: The context instance (passed automatically).
+    """
+    pytest_results = ctx.run(["pytest", "-v"], pty=True)
+    print(pytest_results)
+
+
+@duty
+def coverage(ctx):
+    """
+    Generate a coverage report and save to XML and HTML.
+
+    Args:
+        ctx: The context instance (passed automatically).
+
+    Example:
+        `duty coverage`
+    """
+    ctx.run(["coverage", "run", "--source", PACKAGE_NAME, "-m", "pytest"])
+    res = ctx.run(["coverage", "report"], pty=True)
+    print(res)
+    ctx.run(["coverage", "html"])
+    ctx.run(["coverage", "xml"])
+
+
+@duty
+def bump(ctx, version: str = "patch"):
+    """
+    Bump the version using Poetry and update _version.py.
+
+    This duty is ran as part of `duty release`.
+
+    Args:
+        ctx: The context instance (passed automatically).
+        version (str, optional) = poetry version flag. Available options are:
+            patch, minor, major. Defaults to patch.
+
+    Example:
+        `duty bump version=major`
+    """
+
+    # bump with poetry
+    result = ctx.run(["poetry", "version", version])
+    new_version = re.search(r"(?:.*)(?:\s)(\d+\.\d+\.\d+)$", result)
+    print(new_version.group(0))
+
+    # update _version.py
+    version_file = pathlib.Path(PACKAGE_NAME) / "_version.py"
+    with version_file.open("w", encoding="utf-8") as version_file:
+        version_file.write(
+            f'"""Module containing the version of {PACKAGE_NAME}."""\n\n' + f'__version__ = "{new_version.group(1)}"\n'
+        )
+    print(f"Bumped _version.py to {new_version.group(1)}")
+
+
+@duty
+def build(ctx):
+    """
+    Build with poetry and extract the setup.py and copy to project root.
+
+    Args:
+        ctx: The context instance (passed automatically).
+
+    Example:
+        `duty build`
+    """
+
+    repo_root = pathlib.Path(".")
+
+    # build with poetry
+    result = ctx.run(["poetry", "build"])
+    print(result)
+
+    # extract the setup.py from the tar
+    extracted_tar = re.search(r"(?:.*)(?:Built\s)(.*)", result)
+    tar_file = pathlib.Path(f"./dist/{extracted_tar.group(1)}")
+    shutil.unpack_archive(tar_file, tar_file.parents[0])
+
+    # copy setup.py to repo root
+    extracted_path = tar_file.parents[0] / os.path.splitext(tar_file.stem)[0]
+    setup_py = extracted_path / "setup.py"
+    shutil.copyfile(setup_py, (repo_root / "setup.py"))
+
+    # cleanup
+    shutil.rmtree(extracted_path)
+
+
+@duty
+def release(ctx, version: str = "patch") -> None:
+    """
+    Prepare package for a new release.
+
+    Will run bump, build, export. Manual running of publish is required afterwards.
+
+    Args:
+        ctx: The context instance (passed automatically).
+        version (str): poetry version flag. Available options are: patch, minor, major.
+    """
+    print(ctx.run(["duty", "bump", f"version={version}"]))
+    ctx.run(["duty", "build"])
+    ctx.run(["duty", "export"])
+    print(
+        "✔ Check generated files. Run `duty changelog planned_release= previous_release=` and `duty publish password=`"
+        " when ready to publish."
+    )
+
+
+@duty
+def export(ctx):
+    """
+    Export the dependencies to a requirements.txt file.
+
+    Args:
+        ctx: The context instance (passed automatically).
+
+    Example:
+        `duty export`
+    """
+    requirements_content = ctx.run(
+        [
+            "poetry",
+            "export",
+            "-f",
+            "requirements.txt",
+            "--without-hashes",
+        ]
+    )
+    requirements_dev_content = ctx.run(
+        [
+            "poetry",
+            "export",
+            "-f",
+            "requirements.txt",
+            "--without-hashes",
+            "--dev",
+        ]
+    )
+
+    requirements = pathlib.Path(".") / "requirements.txt"
+    requirements_dev = pathlib.Path(".") / "requirements_dev.txt"
+
+    with requirements.open("w", encoding="utf-8") as req:
+        req.write(requirements_content)
+
+    with requirements_dev.open("w", encoding="utf-8") as req:
+        req.write(requirements_dev_content)
+
+
+@duty
+def publish(ctx, password: str):
+    """
+    Publish the package to pypi.org.
+
+    Args:
+        ctx: The context instance (passed automatically).
+        password (str): pypi.org password.
+
+    Example:
+        `duty publish password=$my_password`
+    """
+    dist_dir = pathlib.Path(".") / "dist"
+    rm_result = rm_tree(dist_dir)
+    print(rm_result)
+
+    publish_result = ctx.run(["poetry", "publish", "-u", "dtomlinson", "-p", password, "--build"])
+    print(publish_result)
+
+
+@duty(silent=True)
+def clean(ctx):
+    """
+    Delete temporary files.
+
+    Args:
+        ctx: The context instance (passed automatically).
+    """
+    ctx.run("rm -rf .mypy_cache")
+    ctx.run("rm -rf .pytest_cache")
+    ctx.run("rm -rf tests/.pytest_cache")
+    ctx.run("rm -rf build")
+    ctx.run("rm -rf dist")
+    ctx.run("rm -rf pip-wheel-metadata")
+    ctx.run("rm -rf site")
+    ctx.run("rm -rf coverage.xml")
+    ctx.run("rm -rf pytest.xml")
+    ctx.run("rm -rf htmlcov")
+    ctx.run("find . -iname '.coverage*' -not -name .coveragerc | xargs rm -rf")
+    ctx.run("find . -type d -name __pycache__ | xargs rm -rf")
+    ctx.run("find . -name '*.rej' -delete")
+
+
+@duty
+def format(ctx):
+    """
+    Format code using Black and isort.
+
+    Args:
+        ctx: The context instance (passed automatically).
+    """
+    res = ctx.run(["black", "--line-length=99", PACKAGE_NAME], pty=True, title="Running Black")
+    print(res)
+
+    res = ctx.run(["isort", PACKAGE_NAME])
+    print(res)
+
+
+@duty(pre=["check_code_quality", "check_types", "check_docs", "check_dependencies"])
+def check(ctx):
+    """
+    Check the code quality, check types, check documentation builds and check dependencies for vulnerabilities.
+
+    Args:
+        ctx: The context instance (passed automatically).
+    """
+
+
+@duty
+def check_code_quality(ctx):
+    """
+    Check the code quality using prospector.
+
+    Args:
+        ctx: The context instance (passed automatically).
+    """
+    ctx.run(["prospector", PACKAGE_NAME], pty=True, title="Checking code quality with prospector")
+
+
+@duty
+def check_types(ctx):
+    """
+    Check the types using mypy.
+
+    Args:
+        ctx: The context instance (passed automatically).
+    """
+    ctx.run(["mypy", PACKAGE_NAME], pty=True, title="Checking types with MyPy")
+
+
+@duty
+def check_docs(ctx):
+    """
+    Check the documentation builds successfully.
+
+    Args:
+        ctx: The context instance (passed automatically).
+    """
+    ctx.run(["mkdocs", "build"], title="Building documentation")
+
+
+@duty
+def check_dependencies(ctx):
+    """
+    Check dependencies with safety for vulnerabilities.
+
+    Args:
+        ctx: The context instance (passed automatically).
+    """
+    for module in sys.modules:
+        if module.startswith("safety.") or module == "safety":
+            del sys.modules[module]
+
+    importlib.invalidate_caches()
+
+    from safety import safety
+    from safety.formatter import report
+    from safety.util import read_requirements
+
+    requirements = ctx.run(
+        "poetry export --dev --without-hashes",
+        title="Exporting dependencies as requirements",
+        allow_overrides=False,
+    )
+
+    def check_vulns():
+        packages = list(read_requirements(StringIO(requirements)))
+        vulns = safety.check(packages=packages, ignore_ids="41002", key="", db_mirror="", cached=False, proxy={})
+        output_report = report(vulns=vulns, full=True, checked_packages=len(packages))
+        print(vulns)
+        if vulns:
+            print(output_report)
+
+    ctx.run(
+        check_vulns,
+        stdin=requirements,
+        title="Checking dependencies",
+        pty=True,
+    )
+
+
+@duty
+def changelog(ctx, planned_release: Optional[str] = None, previous_release: Optional[str] = None):
+    """
+    Generate a changelog with git-cliff.
+
+    Args:
+        ctx: The context instance (passed automatically).
+        planned_release (str, optional): The planned release version. Example: v1.0.2
+        previous_release (str, optional): The previous release version. Example: v1.0.1
+    """
+    generated_changelog: str = ctx.run(["git", "cliff", "-u", "-t", planned_release, "-s", "header"])[:-1]
+    if previous_release is not None:
+        generated_changelog: list = generated_changelog.splitlines()
+        generated_changelog.insert(
+            1,
+            f"<small>[Compare with {previous_release}]({REPO_URL}/compare/{previous_release}..{planned_release})</small>",
+        )
+        generated_changelog: str = "\n".join([line for line in generated_changelog]) + "\n"
+    new_changelog = []
+
+    changelog_file = pathlib.Path(".") / "CHANGELOG.md"
+    with changelog_file.open("r", encoding="utf-8") as changelog_contents:
+        all_lines = changelog_contents.readlines()
+        for line_string in all_lines:
+            regex_string = re.search(r"(<!-- marker -->)", line_string)
+            new_changelog.append(line_string)
+            if isinstance(regex_string, re.Match):
+                new_changelog.append(generated_changelog)
+    with changelog_file.open("w", encoding="utf-8") as changelog_contents:
+        changelog_contents.writelines(new_changelog)
+
+
+def rm_tree(directory: pathlib.Path):
+    """
+    Recursively delete a directory and all its contents.
+
+    Args:
+        directory (pathlib.Path): The directory to delete.
+    """
+    for child in directory.glob("*"):
+        if child.is_file():
+            child.unlink()
+        else:
+            rm_tree(child)
+    directory.rmdir()

panaetius/__init__.py

@@ -1,5 +1,7 @@
 """
-panaetius - a utility library to read variables and provide convenient logging.
+Panaetius package.
+
+A utility library to read variables and provide convenient logging.
 
 Author: Daniel Tomlinson (dtomlinson@panaetius.co.uk)
 """

panaetius/_version.py

@@ -0,0 +1,3 @@
+"""Module containing the version of panaetius."""
+
+__version__ = "2.3.4"

panaetius/config.py

@@ -1,5 +1,5 @@
 """
-Access variables from a config file or an environment variable.
+Config module to access variables from a config file or an environment variable.
 
 This module defines the `Config` class to interact and read variables from either a
 `config.yml` or an environment variable.
@@ -19,7 +19,16 @@ from panaetius.exceptions import KeyErrorTooDeepException
 
 
 class Config:
-    """The configuration class to access variables."""
+    """
+    A configuration class to access user variables.
+
+    Args:
+        header_variable (str): the `header` variable.
+        config_path (str|None=None): the path where the header directory is stored.
+        skip_header_init (bool=False): if True will not use a header subdirectory in the
+            `config_path`.
+
+    """
 
     def __init__(
         self,
@@ -60,17 +69,17 @@ class Config:
     @property
     def config(self) -> dict:
         """
-        Return the contents of the config file. If missing returns an empty dictionary.
+        Return the contents of the config file.
+
+        If no config file is specified then this returns an empty dictionary.
 
         Returns:
-            dict: The contents of the `.yml` loaded as a python dictionary.
+            dict: The contents of the config `.yml` loaded as a python dictionary.
         """
         if self.skip_header_init:
             config_file_location = self.config_path / "config.yml"
         else:
-            config_file_location = (
-                self.config_path / self.header_variable / "config.yml"
-            )
+            config_file_location = self.config_path / self.header_variable / "config.yml"
         try:
             with open(config_file_location, "r", encoding="utf-8") as config_file:
                 # return dict(toml.load(config_file))
@@ -138,15 +147,16 @@ class Config:
             if value is not None:
                 return self.__get_config_value_env_var_override(value)
 
-            if len(key.split(".")) > 2:
+            if len(key.split(".")) > 3:
                 raise KeyErrorTooDeepException(
-                    f"Your key of {key} can only be 2 levels deep maximum. "
-                    f"You have {len(key.split('.'))}"
+                    f"Your key of {key} can only be 3 levels deep maximum."
                 )
             if len(key.split(".")) == 1:
                 return self.__get_config_value_key_split_once(key)
             if len(key.split(".")) == 2:
                 return self.__get_config_value_key_split_twice(key)
+            if len(key.split(".")) == 3:
+                return self.__get_config_value_key_split_thrice(key)
             raise KeyError()
 
         except (KeyError, TypeError):
@@ -163,6 +173,10 @@ class Config:
         section, name = key.lower().split(".")
         return self.config[self.header_variable][section][name]
 
+    def __get_config_value_key_split_thrice(self, key: str) -> Any:
+        section, name_0, name_1 = key.lower().split(".")
+        return self.config[self.header_variable][section][name_0][name_1]
+
     def __get_config_value_missing_key_value_is_none(self, default: Any) -> Any:
         return self.__load_default_value(default)
 

panaetius/exceptions.py

@@ -1,13 +1,13 @@
-"""Exceptions for the module."""
+"""Module that defines custom exceptions for Panetius."""
 
 
 class KeyErrorTooDeepException(Exception):
-    pass
+    """Raised if the keys in the config.yml are nested too deeply."""
 
 
 class LoggingDirectoryDoesNotExistException(Exception):
-    pass
+    """Raised if the logging directory does not exist."""
 
 
 class InvalidPythonException(Exception):
-    pass
+    """Raised if the environement variable Python type is invalid."""

panaetius/logging.py

@@ -63,11 +63,17 @@ def set_logger(config_inst: Config, logging_format_inst: LoggingData) -> logging
 
     # configure file handler
     if config_inst.logging_path is not None:
-        logging_file = (
-            pathlib.Path(config_inst.logging_path)
-            / config_inst.header_variable
-            / f"{config_inst.header_variable}.log"
-        ).expanduser()
+        if not config_inst.skip_header_init:
+            logging_file = (
+                pathlib.Path(config_inst.logging_path)
+                / config_inst.header_variable
+                / f"{config_inst.header_variable}.log"
+            ).expanduser()
+        else:
+            logging_file = (
+                pathlib.Path(config_inst.logging_path)
+                / f"{config_inst.header_variable}.log"
+            ).expanduser()
 
         if not logging_file.parents[0].exists():
             raise LoggingDirectoryDoesNotExistException()
@@ -100,6 +106,11 @@ class LoggingData(metaclass=ABCMeta):
     def format(self) -> str:
         raise NotImplementedError
 
+    @property
+    @abstractmethod
+    def logging_level(self) -> str:
+        raise NotImplementedError
+
     @abstractmethod
     def __init__(self, logging_level: str):
         raise NotImplementedError
@@ -113,8 +124,12 @@ class SimpleLogger(LoggingData):
             '"%(levelname)s",\n\t"message": "%(message)s"\n}',
         )
 
+    @property
+    def logging_level(self) -> str:
+        return self._logging_level
+
     def __init__(self, logging_level: str = "INFO"):
-        self.logging_level = logging_level
+        self._logging_level = logging_level
 
 
 class AdvancedLogger(LoggingData):
@@ -127,8 +142,12 @@ class AdvancedLogger(LoggingData):
             '"%(levelname)s",\n\t"message": "%(message)s"\n}',
         )
 
+    @property
+    def logging_level(self) -> str:
+        return self._logging_level
+
     def __init__(self, logging_level: str = "INFO"):
-        self.logging_level = logging_level
+        self._logging_level = logging_level
 
 
 class CustomLogger(LoggingData):
@@ -136,6 +155,10 @@ class CustomLogger(LoggingData):
     def format(self) -> str:
         return str(self._format)
 
+    @property
+    def logging_level(self) -> str:
+        return self._logging_level
+
     def __init__(self, logging_format: str, logging_level: str = "INFO"):
-        self.logging_level = logging_level
+        self._logging_level = logging_level
         self._format = logging_format

panaetius/utilities/__init__.py

@@ -1,3 +1,3 @@
-"""General utilities."""
+"""Sub-package which defines general utility functions."""
 
 from panaetius.utilities.squasher import Squash

panaetius/utilities/squasher.py

@@ -1,4 +1,4 @@
-"""Squash a json object or Python dictionary into a single level dictionary."""
+"""Sub-module that defines squashing json objects into a single json object."""
 
 from __future__ import annotations
 

pyproject.toml

@@ -1,10 +1,10 @@
 [tool.poetry]
 name = "panaetius"
-version = "2.2.1"
+version = "2.3.4"
 description = "Python module to gracefully handle a .config file/environment variables for scripts, with built in masking for sensitive options. Provides a Splunk friendly formatted logger instance."
 license = "MIT"
 authors = ["dtomlinson <dtomlinson@panaetius.co.uk>"]
-readme = "./README.rst"
+readme = "./README.md"
 homepage = "https://github.com/dtomlinson91/panaetius"
 repository = "https://github.com/dtomlinson91/panaetius"
 documentation = "https://panaetius.readthedocs.io/en/latest/introduction.html"
@@ -34,6 +34,24 @@ pytest = "^6.2.5"
 pytest-datadir = "^1.3.1"
 pytest-xdist = "^2.4.0"
 coverage = "^6.0.2"
+duty = "^0.7.0"
+types-PyYAML = "^6.0.1"
+isort = "^5.10.1"
+mypy = "^0.910"
+safety = "^1.10.3"
+
+[tool.black]
+line-length = 120
+
+[tool.isort]
+line-length = 120
+not_skip = "__init__.py"
+multi_line_output = 3
+force_single_line = false
+balanced_wrapping = true
+default_section = "THIRDPARTY"
+known_first_party = "duty"
+include_trailing_comma = true
 
 [build-system]
 requires = ["poetry>=0.12"]

requirements.txt

@@ -1,7 +1,2 @@
-pylite==0.1.0 \
-    --hash=sha256:e338d20d3f8f72dd84d1e58f2fd6dba008d593e0cfacfb5fbdd5a297b830628e \
-    --hash=sha256:eb46f5beb1f2102672fd4355c013ac2feebc0df284d65f7711f2041a0a410141
-toml==0.10.0 \
-    --hash=sha256:229f81c57791a41d65e399fc06bf0848bab550a9dfd5ed66df18ce5f05e73d5c \
-    --hash=sha256:235682dd292d5899d361a811df37e04a8828a5b1da3115886b73cf81ebc9100e \
-    --hash=sha256:f1db651f9657708513243e61e6cc67d101a39bad662eaa9b5546f789338e07a3
+pyyaml==6.0; python_version >= "3.6"
+toml==0.10.2; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.3.0")

requirements_dev.txt

@@ -0,0 +1,68 @@
+ansimarkup==1.5.0; python_version >= "3.6"
+astroid==2.9.1; python_full_version >= "3.6.2" and python_version < "4.0"
+atomicwrites==1.4.0; python_version >= "3.6" and python_full_version < "3.0.0" and sys_platform == "win32" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6") or sys_platform == "win32" and python_version >= "3.6" and python_full_version >= "3.4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6")
+attrs==21.4.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
+bandit==1.7.1; python_full_version >= "3.6.1" and python_version < "4.0" and python_version >= "3.5"
+cached-property==1.5.2; python_version < "3.8" and python_version >= "3.6"
+certifi==2021.10.8; python_version >= "3.5" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.5"
+charset-normalizer==2.0.9; python_full_version >= "3.6.0" and python_version >= "3.5"
+click==8.0.3; python_version >= "3.6"
+colorama==0.4.4; sys_platform == "win32" and python_version >= "3.6" and python_full_version >= "3.6.2" and python_version < "4.0" and platform_system == "Windows" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6")
+coverage==6.2; python_version >= "3.6"
+dodgy==0.2.1; python_full_version >= "3.6.1" and python_version < "4.0"
+dparse==0.5.1; python_version >= "3.5"
+duty==0.7.0; python_version >= "3.6"
+execnet==1.9.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
+failprint==0.8.0; python_version >= "3.6"
+flake8-polyfill==1.0.2; python_full_version >= "3.6.1" and python_version < "4.0"
+flake8==2.3.0; python_full_version >= "3.6.1" and python_version < "4.0"
+gitdb==4.0.9; python_full_version >= "3.6.1" and python_version < "4.0" and python_version >= "3.7"
+gitpython==3.1.24; python_full_version >= "3.6.1" and python_version < "4.0" and python_version >= "3.7"
+idna==3.3; python_version >= "3.5" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.5"
+importlib-metadata==4.10.0; python_version < "3.8" and python_version >= "3.7" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6") and python_full_version >= "3.6.1"
+iniconfig==1.1.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
+isort==5.10.1; python_full_version >= "3.6.1" and python_version < "4.0"
+jinja2==3.0.3; python_version >= "3.6"
+lazy-object-proxy==1.7.1; python_full_version >= "3.6.2" and python_version < "4.0" and python_version >= "3.6"
+markupsafe==2.0.1; python_version >= "3.6"
+mccabe==0.6.1; python_full_version >= "3.6.2" and python_version < "4.0"
+mypy-extensions==0.4.3; python_full_version >= "3.6.1" and python_version < "4.0" and python_version >= "3.5"
+mypy==0.910; python_version >= "3.5"
+packaging==21.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
+pbr==5.8.0; python_full_version >= "3.6.1" and python_version < "4.0" and python_version >= "3.6"
+pep8-naming==0.10.0; python_full_version >= "3.6.1" and python_version < "4.0"
+pep8==1.7.1; python_full_version >= "3.6.1" and python_version < "4.0"
+platformdirs==2.4.1; python_full_version >= "3.6.2" and python_version < "4.0" and python_version >= "3.7"
+pluggy==1.0.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
+prospector==1.5.3.1; python_full_version >= "3.6.1" and python_version < "4.0"
+ptyprocess==0.7.0; sys_platform != "win32" and python_version >= "3.6"
+py==1.11.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
+pycodestyle==2.8.0; python_full_version >= "3.6.1" and python_version < "4.0"
+pydocstyle==6.1.1; python_full_version >= "3.6.1" and python_version < "4.0" and python_version >= "3.6"
+pyflakes==2.3.1; python_full_version >= "3.6.1" and python_version < "4.0"
+pylint-celery==0.3; python_full_version >= "3.6.1" and python_version < "4.0"
+pylint-django==2.4.4; python_full_version >= "3.6.1" and python_version < "4.0"
+pylint-flask==0.6; python_full_version >= "3.6.1" and python_version < "4.0"
+pylint-plugin-utils==0.6; python_full_version >= "3.6.1" and python_version < "4.0"
+pylint==2.12.2; python_full_version >= "3.6.2" and python_version < "4.0"
+pyparsing==3.0.6; python_version >= "3.6"
+pytest-datadir==1.3.1; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0")
+pytest-forked==1.4.0; python_version >= "3.6"
+pytest-xdist==2.5.0; python_version >= "3.6"
+pytest==6.2.5; python_version >= "3.6"
+pyyaml==6.0; python_version >= "3.6"
+requests==2.26.0; python_version >= "3.5" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.5"
+requirements-detector==0.7; python_full_version >= "3.6.1" and python_version < "4.0"
+safety==1.10.3; python_version >= "3.5"
+setoptconf-tmp==0.3.1; python_full_version >= "3.6.1" and python_version < "4.0"
+smmap==5.0.0; python_full_version >= "3.6.1" and python_version < "4.0" and python_version >= "3.7"
+snowballstemmer==2.2.0; python_full_version >= "3.6.1" and python_version < "4.0" and python_version >= "3.6"
+stevedore==3.5.0; python_full_version >= "3.6.1" and python_version < "4.0" and python_version >= "3.6"
+toml==0.10.2; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.3.0")
+typed-ast==1.4.3; python_version < "3.8" and python_version >= "3.5" and python_full_version >= "3.6.2" and implementation_name == "cpython"
+types-pyyaml==6.0.1
+types-toml==0.10.1
+typing-extensions==4.0.1; python_full_version >= "3.6.2" and python_version < "3.8" and python_version >= "3.7"
+urllib3==1.26.7; python_version >= "3.5" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version < "4" and python_version >= "3.5"
+wrapt==1.13.3; python_full_version >= "3.6.2" and python_version < "4.0"
+zipp==3.7.0; python_version < "3.8" and python_version >= "3.7"

rewrite.todo

@@ -1,53 +0,0 @@
-Testing:
-  To Write:
-    ☐ Test the Config file skipping header with `skip_header_init`
-    ☐ Document coverage commands
-      `coverage run --source=./panaetius -m pytest`
-      `coverage report` & `coverage html` > gives ./htmlcov/index.html
-    ☐ Document for abstract methods should raise NotImplementedError
-    ☐ Document https://stackoverflow.com/a/9212387
-Documentation:
-    ☐ Rewrite documentation using `mkdocs` and using `.md`.
-    ☐ Update the metadata in the `pyproject.toml`.
-    ☐ Create a new `Readme.md` and remove the `.rst`.
-    ☐ Document the logging strategy
-      CLI tools should use `logger.critical` and raise SystemExit(1)
-      Libraries should raise custom errors and have a `logger.critical(exec_info=1)`
-
-Misc:
-    ☐ Use the python runner to build the docs & run the tests (including coverage html)
-        coverage run -m pytest && coverage report && coverage html
-        ☐ document this in trilium
-
-Archive:
-  ✘ Bump the version to release 2.0 @cancelled(21-10-23 05:36) @project(Misc)
-  ✔ Handle if a bool is passed in as a default @done(21-10-16 05:25) @project(Coding.No Config File)
-  ✔ Handle if a bool is passed in as a default @done(21-10-16 05:25) @project(Coding.Config File)
-  ✔ Create SimpleLogger, AdvancedLogger, CustomLogger classes @done(21-10-16 16:22) @project(Coding.Logging)
-  ✔ Logging path should take by default the config path unless overwritten? @done(21-10-16 23:49) @project(Coding.Logging)
-  ✔ Check logging path + config path are valid, if not raise error. @done(21-10-18 00:04) @project(Coding.Errors)
-  ✔ Add tests for these. @done(21-10-18 00:04) @project(Coding.Errors)
-  ✔ Check for a key > 2 levels, raise custom error, write test @done(21-10-17 23:30) @project(Coding.Errors)
-  ✔ Check all functions and annotations. @done(21-10-18 01:07) @project(Coding.Linting)
-  ✔ Write the docstrings for public functions/methods. @done(21-10-18 02:29) @project(Coding.Docstrings)
-  ✔ When both a config file and a env var is found, use the env var. @done(21-10-18 00:38) @project(Coding.Functionality)
-  ✔ If loading from a default, don't covert to TOML @done(21-10-17 20:33) @project(Tests.Bugfixes)
-  ✔ Env Vars should be given as python objects @done(21-10-17 20:33) @project(Tests.Bugfixes)
-    The string or node provided may only consist of the following Python literal structures: strings, bytes, numbers, tuples, lists, dicts, sets, booleans, and None.
-    use ast.literal_eval()
-    https://docs.python.org/3/library/ast.html#ast.literal_eval
-  ✔ Test default config path set to "~/.config" @done(21-10-17 17:25) @project(Tests.__init__)
-  ✔ Test config path is set when passed in @done(21-10-17 17:25) @project(Tests.__init__)
-  ✔ Check testing config file is returned as dict @done(21-10-17 17:25) @project(Tests.config property)
-  ✔ Check _self.missing_config and empty dict is returned @done(21-10-17 17:25) @project(Tests.config property)
-  ✔ Arrays & tables loaded correctly from config file @done(21-10-17 20:34) @project(Tests.get_value.config_file)
-  ✔ test when key length is 1 the value is returned @done(21-10-17 18:55) @project(Tests.get_value.config_file)
-  ✔ test when key length is 2 the value is returned @done(21-10-17 18:55) @project(Tests.get_value.config_file)
-  ✔ test when key not found and no env var default is loaded @done(21-10-17 19:01) @project(Tests.get_value.config_file)
-  ✔ test bool's are properly converted @done(21-10-17 19:01) @project(Tests.get_value.config_file)
-  ✔ test when key not found and env var is set value is loaded @done(21-10-17 20:43) @project(Tests.get_value.config_file)
-  ✔ check if env key is missing the default is read in @done(21-10-17 20:55) @project(Tests.get_value.env_var)
-  ✔ check if env key is present the values are read in @done(21-10-17 22:24) @project(Tests.get_value.env_var)
-  ✔ parametrise a test to read in values form env vars and they're set correctly @done(21-10-17 22:24) @project(Tests.get_value.env_var)
-  ✔ test that the env var is valid python @done(21-10-18 01:03) @project(Tests.get_value.env_var)
-  ✔ test set_config works @done(21-10-17 23:29) @project(Tests.library)

setup.py

@@ -12,9 +12,9 @@ install_requires = \
 
 setup_kwargs = {
     'name': 'panaetius',
-    'version': '2.2.1',
+    'version': '2.3.4',
     'description': 'Python module to gracefully handle a .config file/environment variables for scripts, with built in masking for sensitive options. Provides a Splunk friendly formatted logger instance.',
-    'long_description': 'Author\n=======\n\nDaniel Tomlinson (dtomlinson@panaetius.co.uk)\n\nRequires\n=========\n\n`>= python3.7`\n\nPython requirements\n====================\n\n- toml = "^0.10.0"\n- pylite = "^0.1.0"\n\nDocumentation\n==============\n\nRead the documentation on `read the docs`_.\n\n.. _read the docs: https://panaetius.readthedocs.io/en/latest/introduction.html\n\nInstallation\n==============\n\nYou can install ``panaetius`` the following ways:\n\nPython\n-------\n\n.. Attention:: You should install in a python virtual environment\n\nFrom pypi using pip\n~~~~~~~~~~~~~~~~~~~~\n\n.. code-block:: bash\n\n    pip install panaetius\n\nFrom local wheel\n~~~~~~~~~~~~~~~~~\n\nDownload the latest verion from the `releases`_ page.\n\n.. _releases: https://github.com/dtomlinson91/panaetius/releases\n\nInstall with pip:\n\n.. code-block:: bash\n\n    pip install -U panaetius-1.0.2-py3-none-any.whl\n\n\nFrom source\n~~~~~~~~~~~~\n\nClone the repo and install using ``setup.py``:\n\n.. code-block:: bash\n\n    python setup.py\n',
+    'long_description': '# Panaetius\n\nThis package provides:\n\n- Functionality to read user variables from a `config.yml` or environment variables.\n- A convenient default logging formatter printing `json` that can save to disk and rotate.\n- Utility functions.\n\n## Config\n\n### options\n\n#### skip_header_init\n\nIf `skip_header_init=True` then the `config_path` will not use the `header_variable` as the\nsub-directory in the `config_path`.\n\nE.g\n\n`CONFIG = panaetius.Config("tembo", "~/tembo/.config", skip_header_init=True)`\n\nWill look in `~/tembo/config/config.yml`.\n\nIf `skip_header_init=False` then would look in `~/tembo/config/tembo/config.yml`.\n\n### Module\n\nConvenient to place in a package/sub-package `__init__.py`.\n\nSee Tembo for an example: <https://github.com/tembo-pages/tembo-core/blob/main/tembo/cli/__init__.py>\n\nExample snippet to use in a module:\n\n```python\n"""Subpackage that contains the CLI application."""\n\nimport os\nfrom typing import Any\n\nimport panaetius\nfrom panaetius.exceptions import LoggingDirectoryDoesNotExistException\n\n\nif (config_path := os.environ.get("TEMBO_CONFIG")) is not None:\n    CONFIG: Any = panaetius.Config("tembo", config_path, skip_header_init=True)\nelse:\n    CONFIG = panaetius.Config(\n        "tembo", "~/tembo/.config", skip_header_init=True\n    )\n\n\npanaetius.set_config(CONFIG, "base_path", "~/tembo")\npanaetius.set_config(CONFIG, "template_path", "~/tembo/.templates")\npanaetius.set_config(CONFIG, "scopes", {})\npanaetius.set_config(CONFIG, "logging.level", "DEBUG")\npanaetius.set_config(CONFIG, "logging.path")\n\ntry:\n    logger = panaetius.set_logger(\n        CONFIG, panaetius.SimpleLogger(logging_level=CONFIG.logging_level)\n    )\nexcept LoggingDirectoryDoesNotExistException:\n    _LOGGING_PATH = CONFIG.logging_path\n    CONFIG.logging_path = ""\n    logger = panaetius.set_logger(\n        CONFIG, panaetius.SimpleLogger(logging_level=CONFIG.logging_level)\n    )\n    logger.warning("Logging directory %s does not exist", _LOGGING_PATH)\n\n```\n\nThis means in `./tembo/cli/cli.py` you can\n\n```python\nimport tembo.cli\n\n# access the CONFIG instance + variables from the config.yml\ntembo.cli.CONFIG\n```\n\n### Script\n\nCreate `./config/config.yml` in the same directory as the script.\n\nIn the script initialise a `CONFIG` object:\n\n```python\nimport pathlib\n\nimport panaetius\n\nCONFIG = panaetius.Config(\n    "teenagers_scraper", str(pathlib.Path(__file__).parents[0] / ".config"), skip_header_init=True\n)\n```\n\nSet variables in the same way as the module above.\n\n#### quickstart logging\n\n```python\nimport panaetius\n\n\ndef get_logger():\n    logging_dir = pathlib.Path(__file__).parents[0] / "logs"\n    logging_dir.mkdir(parents=True, exist_ok=True)\n\n    CONFIG = panaetius.Config("training_data_into_gcp", skip_header_init=True)\n    panaetius.set_config(CONFIG, "logging.level", "DEBUG")\n    panaetius.set_config(CONFIG, "logging.path", logging_dir)\n    return panaetius.set_logger(CONFIG, panaetius.SimpleLogger(logging_level=CONFIG.logging_level))\n```\n\n## Utility Functions\n\n### Squasher\n\nSquashes a json object or Python dictionary into a single level dictionary.\n',
     'author': 'dtomlinson',
     'author_email': 'dtomlinson@panaetius.co.uk',
     'maintainer': None,

tests/conftest.py

@@ -18,6 +18,9 @@ def testing_config_contents():
                 "some_second_list": ["some", "second", "value"],
                 "some_second_table": {"first": ["some", "first", "value"]},
                 "some_second_table_bools": {"bool": [True, False]},
+                "third": {
+                    "some_third_string": "some_third_value",
+                },
             },
         }
     }

tests/data/without_logging/panaetius_testing/config.yml

@@ -7,3 +7,5 @@
     some_second_list: ["some", "second", "value"]
     some_second_table: { "first": ["some", "first", "value"] }
     some_second_table_bools: { "bool": [true, false] }
+    third:
+      some_third_string: some_third_value

tests/test_config.py

@@ -67,9 +67,7 @@ def test_config_file_without_header_dir_exists(header, shared_datadir):
     assert config._missing_config is False
 
 
-def test_config_file_contents_read_success(
-    header, shared_datadir, testing_config_contents
-):
+def test_config_file_contents_read_success(header, shared_datadir, testing_config_contents):
     # arrange
     config_path = str(shared_datadir / "without_logging")
 
@@ -101,11 +99,10 @@ def test_config_file_contents_read_success(
             "second_some_second_table_bools",
             {"bool": [True, False]},
         ),
+        ("second.third.some_third_string", "second_third_some_third_string", "some_third_value"),
     ],
 )
-def test_get_value_from_key(
-    set_config_key, get_config_key, expected_value, header, shared_datadir
-):
+def test_get_value_from_key(set_config_key, get_config_key, expected_value, header, shared_datadir):
     """
     Test the following:
 
@@ -155,11 +152,7 @@ def test_key_level_too_deep(header, shared_datadir):
         panaetius.set_config(config, key)
 
     # assert
-    assert (
-        str(key_error_too_deep.value)
-        == f"Your key of {key} can only be 2 levels deep maximum. "
-        f"You have 4"
-    )
+    assert str(key_error_too_deep.value) == f"Your key of {key} can only be 3 levels deep maximum."
 
 
 def test_get_value_missing_key_from_default(header, shared_datadir):
@@ -242,9 +235,7 @@ def test_missing_config_read_from_default(header, shared_datadir):
         ),
     ],
 )
-def test_missing_config_read_from_env_var(
-    env_value, expected_value, header, shared_datadir
-):
+def test_missing_config_read_from_env_var(env_value, expected_value, header, shared_datadir):
     # arrange
     config_path = str(shared_datadir / str(uuid4()))
     os.environ[f"{header.upper()}_MISSING_KEY_READ_FROM_ENV_VAR"] = env_value
@@ -271,10 +262,7 @@ def test_missing_config_read_from_env_var_invalid_python(header):
         panaetius.set_config(config, "invalid_python")
 
     # assert
-    assert (
-        str(invalid_python_exception.value)
-        == "a string without quotes is not valid Python."
-    )
+    assert str(invalid_python_exception.value) == "a string without quotes is not valid Python."
 
     # cleanup
     del os.environ[f"{header.upper()}_INVALID_PYTHON"]