add todo

* linting

* adding py.typed

* adding pyyaml types

* workaround #1

* updating latest

* updating dev dependencies

* adding isort mypy safety

* chore: prepare release 2.3.3

.gitignore

@@ -130,3 +130,4 @@ dmypy.json
 
 # custom
 .DS_Store
+.vscode/*

.vscode/settings.json

@@ -1,23 +0,0 @@
-{
-    "python.linting.pylintEnabled": false,
-    "python.linting.prospectorEnabled": true,
-    "python.linting.enabled": true,
-    "python.pythonPath": ".venv/bin/python",
-    "restructuredtext.confPath": "${workspaceFolder}/docs/source",
-    "peacock.color": "#307E6A",
-    "workbench.colorCustomizations": {
-        "editorGroup.border": "#3ea389",
-        "panel.border": "#3ea389",
-        "sash.hoverBorder": "#3ea389",
-        "sideBar.border": "#3ea389",
-        "statusBar.background": "#307e6a",
-        "statusBar.foreground": "#e7e7e7",
-        "statusBarItem.hoverBackground": "#3ea389",
-        "statusBarItem.remoteBackground": "#307e6a",
-        "statusBarItem.remoteForeground": "#e7e7e7",
-        "titleBar.activeBackground": "#307e6a",
-        "titleBar.activeForeground": "#e7e7e7",
-        "titleBar.inactiveBackground": "#307e6a99",
-        "titleBar.inactiveForeground": "#e7e7e799"
-    }
-}

CHANGELOG.md

@@ -0,0 +1,30 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+<!-- marker -->
+## [v2.3.5](https://github.com/dtomlinson91/panaetius/commits/v2.3.5) - 2022-01-30
+<small>[Compare with v2.3.4](https://github.com/dtomlinson91/panaetius/compare/v2.3.4...v2.3.5)</small>
+
+### 🧱 Build
+
+- Relax PyYAML version constraint for better compatibility ([a0583b5](https://github.com/dtomlinson91/panaetius/commit/a0583b5f0aa3068139827ff46f8a7aa16cd6b424))
+## [v2.3.4](https://github.com/dtomlinson91/panaetius/commits/v2.3.4) - 2022-01-03
+<small>[Compare with 2.3.3](https://github.com/dtomlinson91/panaetius/compare/2.3.3...v2.3.4)</small>
+
+### ✨ Features
+
+- Add ability to retrieve keys 3 levels deep ([f5ea19e](https://github.com/dtomlinson91/panaetius/commit/f5ea19e7d2f977244594b378c6b7633f02f6048a))
+
+### 📘 Documentation
+
+- Update README.md with script boilerplate ([1790071](https://github.com/dtomlinson91/panaetius/commit/1790071741207de13330ba75d7bf090106290d72))
+- Update README.md with script quickstart logging ([e6cfded](https://github.com/dtomlinson91/panaetius/commit/e6cfded87dcfc5d2bf62d36bc7b4dbbdeb94b0b8))
+
+### 🧪 Testing
+
+- Add tests for f5ea19e ([156af46](https://github.com/dtomlinson91/panaetius/commit/156af4685510bac97a850b83d63f8337635db199))
+
+### 🧱 Build
+
+- Update dependencies ([9bc89fd](https://github.com/dtomlinson91/panaetius/commit/9bc89fd2ce9ddf8dcd6a3ca84ef9b72ee183efd3))

README.md

@@ -1,5 +1,8 @@
 # Panaetius
 
+![PyPI](https://img.shields.io/pypi/v/panaetius)
+![PyPI - Python Version](https://img.shields.io/pypi/pyversions/panaetius)
+
 This package provides:
 
 - Functionality to read user variables from a `config.yml` or environment variables.
@@ -32,16 +35,21 @@ See Tembo for an example: <https://github.com/tembo-pages/tembo-core/blob/main/t
 Example snippet to use in a module:
 
 ```python
+"""Subpackage that contains the CLI application."""
+
 import os
+from typing import Any
 
 import panaetius
 from panaetius.exceptions import LoggingDirectoryDoesNotExistException
 
 
 if (config_path := os.environ.get("TEMBO_CONFIG")) is not None:
-    CONFIG = panaetius.Config("tembo", config_path, skip_header_init=True)
+    CONFIG: Any = panaetius.Config("tembo", config_path, skip_header_init=True)
 else:
-    CONFIG = panaetius.Config("tembo", "~/tembo/.config", skip_header_init=True)
+    CONFIG = panaetius.Config(
+        "tembo", "~/tembo/.config", skip_header_init=True
+    )
 
 
 panaetius.set_config(CONFIG, "base_path", "~/tembo")
@@ -73,6 +81,51 @@ import tembo.cli
 tembo.cli.CONFIG
 ```
 
+### Script
+
+#### with config file
+
+Create `./config/config.yml` in the same directory as the script.
+
+In the script initialise a `CONFIG` object:
+
+```python
+import pathlib
+
+import panaetius
+
+CONFIG = panaetius.Config(
+    "teenagers_scraper", str(pathlib.Path(__file__).parents[0] / ".config"), skip_header_init=True
+)
+```
+
+Set variables in the same way as the module above.
+
+####  with env vars only
+
+```python
+import panaetius
+
+CONFIG = panaetius.Config("mart")
+panaetius.set_config(CONFIG, "username")
+panaetius.set_config(CONFIG, "password")
+```
+
+#### quickstart logging
+
+```python
+import panaetius
+
+
+def get_logger():
+    logging_dir = pathlib.Path(__file__).parents[0] / "logs"
+    logging_dir.mkdir(parents=True, exist_ok=True)
+
+    CONFIG = panaetius.Config("training_data_into_gcp", skip_header_init=True)
+    panaetius.set_config(CONFIG, "logging.level", "DEBUG")
+    panaetius.set_config(CONFIG, "logging.path", logging_dir)
+    return panaetius.set_logger(CONFIG, panaetius.SimpleLogger(logging_level=CONFIG.logging_level))
+```
 
 ## Utility Functions
 

cliff.toml

@@ -0,0 +1,58 @@
+# configuration file for git-cliff (0.1.0)
+
+[changelog]
+# changelog header
+header = """
+# Changelog
+All notable changes to this project will be documented in this file.\n
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).\n
+"""
+# template for the changelog body
+# https://tera.netlify.app/docs/#introduction
+body = """
+{% if version %}\
+    ## [{{ version }}](https://github.com/dtomlinson91/panaetius/commits/{{ version }}) - {{ timestamp | date(format="%Y-%m-%d") }}
+    {% if previous.version %}\
+    <small>[Compare with {{ previous.version }}](https://github.com/dtomlinson91/panaetius/compare/{{ previous.version }}...{{ version }})</small>
+    {% endif %}\
+{% else %}\
+    ## [unreleased]
+{% endif %}\
+{% for group, commits in commits | group_by(attribute="group") %}
+    ### {{ group | upper_first }}
+    {% for commit in commits %}
+        - {{ commit.message | upper_first }} ([{{ commit.id | truncate(length=7, end="") }}](https://github.com/dtomlinson91/panaetius/commit/{{ commit.id }}))\
+    {% endfor %}
+{% endfor %}\n
+"""
+# remove the leading and trailing whitespaces from the template
+trim = true
+# changelog footer
+footer = """
+"""
+
+[git]
+# allow only conventional commits
+# https://www.conventionalcommits.org
+conventional_commits = true
+# regex for parsing and grouping commits
+commit_parsers = [
+    { message = "^feat", group = "✨ Features"},
+    { message = "^fix", group = "🐛 Bug Fixes"},
+    { message = "^doc", group = "📘 Documentation"},
+    { message = "^perf", group = "🏎 Performance"},
+    { message = "^refactor", group = "🛠 Refactor/Improvement"},
+    { message = "^style", group = "🎨 Styling"},
+    { message = "^test", group = "🧪 Testing"},
+    { message = "^build", group = "🧱 Build"},
+    { body = ".*security", group = "🔐 Security"},
+    { message = "^chore\\(release\\): prepare for", skip = true},
+    { message = "^chore", group = "🥱 Miscellaneous Tasks", skip = true},
+]
+# filter out the commits that are not matched by commit parsers
+filter_commits = false
+# glob pattern for matching git tags
+tag_pattern = "v[0-9]*"
+# regex for skipping tags
+skip_tags = "v0.1.0-beta.1"

docs/todo.todo

@@ -0,0 +1,16 @@
+Features:
+    ☐ Add simple one-line logger
+    ☐ Try Typer
+        <https://typer.tiangolo.com/>
+
+
+One-line logger
+```
+LOGGER = panaetius.set_logger(
+    CONFIG,
+    panaetius.CustomLogger(
+        logging_format='{"time": "%(asctime)s", "logging_level":"%(levelname)s", "message": "%(message)s"}',
+        logging_level=CONFIG.logging_level,
+    ),
+)
+```

duties.py

@@ -1,21 +1,31 @@
 from __future__ import annotations
 
+import importlib
 import os
 import pathlib
 import re
 import shutil
+import sys
+from io import StringIO
+from typing import List, Optional, Pattern
+from urllib.request import urlopen
 
 from duty import duty
 
-
 PACKAGE_NAME = "panaetius"
+REPO_URL = "https://github.com/dtomlinson91/panaetius"
 
 
-@duty
+@duty(post=["export"])
 def update_deps(ctx, dry: bool = False):
     """
     Update the dependencies using Poetry.
 
+    Args:
+        ctx: The context instance (passed automatically).
+        dry (bool, optional) = If True will update the `poetry.lock` without updating the
+            dependencies themselves. Defaults to False.
+
     Example:
         `duty update_deps dry=False`
     """
@@ -28,39 +38,52 @@ def update_deps(ctx, dry: bool = False):
 
 @duty
 def test(ctx):
-    """Run tests using pytest"""
-    pytest_results = ctx.run(["pytest", "-v"])
+    """
+    Run tests using pytest.
+
+    Args:
+        ctx: The context instance (passed automatically).
+    """
+    pytest_results = ctx.run(["pytest", "-v"], pty=True)
     print(pytest_results)
 
 
 @duty
 def coverage(ctx):
     """
-    Generate a coverage HTML report.
+    Generate a coverage report and save to XML and HTML.
+
+    Args:
+        ctx: The context instance (passed automatically).
 
     Example:
         `duty coverage`
     """
     ctx.run(["coverage", "run", "--source", PACKAGE_NAME, "-m", "pytest"])
+    res = ctx.run(["coverage", "report"], pty=True)
+    print(res)
     ctx.run(["coverage", "html"])
+    ctx.run(["coverage", "xml"])
 
 
 @duty
-def version(ctx, bump: str = "patch"):
+def bump(ctx, version: str = "patch"):
     """
     Bump the version using Poetry and update _version.py.
 
+    This duty is ran as part of `duty release`.
+
     Args:
-        bump (str, optional) = poetry version flag. Available options are:
-            patch, minor, major, prepatch, preminor, premajor, prerelease.
-            Defaults to patch.
+        ctx: The context instance (passed automatically).
+        version (str, optional) = poetry version flag. Available options are:
+            patch, minor, major. Defaults to patch.
 
     Example:
-        `duty version bump=major`
+        `duty bump version=major`
     """
 
     # bump with poetry
-    result = ctx.run(["poetry", "version", bump])
+    result = ctx.run(["poetry", "version", version])
     new_version = re.search(r"(?:.*)(?:\s)(\d+\.\d+\.\d+)$", result)
     print(new_version.group(0))
 
@@ -68,8 +91,7 @@ def version(ctx, bump: str = "patch"):
     version_file = pathlib.Path(PACKAGE_NAME) / "_version.py"
     with version_file.open("w", encoding="utf-8") as version_file:
         version_file.write(
-            f'"""Module containing the version of {PACKAGE_NAME}."""\n\n'
-            + f'__version__ = "{new_version.group(1)}"\n'
+            f'"""Module containing the version of {PACKAGE_NAME}."""\n\n' + f'__version__ = "{new_version.group(1)}"\n'
         )
     print(f"Bumped _version.py to {new_version.group(1)}")
 
@@ -77,7 +99,10 @@ def version(ctx, bump: str = "patch"):
 @duty
 def build(ctx):
     """
-    Build with poetry and extract the `setup.py` and copy to project root.
+    Build with poetry and extract the setup.py and copy to project root.
+
+    Args:
+        ctx: The context instance (passed automatically).
 
     Example:
         `duty build`
@@ -103,11 +128,34 @@ def build(ctx):
     shutil.rmtree(extracted_path)
 
 
+@duty
+def release(ctx, version: str = "patch") -> None:
+    """
+    Prepare package for a new release.
+
+    Will run bump, build, export. Manual running of publish is required afterwards.
+
+    Args:
+        ctx: The context instance (passed automatically).
+        version (str): poetry version flag. Available options are: patch, minor, major.
+    """
+    print(ctx.run(["duty", "bump", f"version={version}"]))
+    ctx.run(["duty", "build"])
+    ctx.run(["duty", "export"])
+    print(
+        "✔ Check generated files. Run `duty changelog planned_release= previous_release=` and `duty publish password=`"
+        " when ready to publish."
+    )
+
+
 @duty
 def export(ctx):
     """
     Export the dependencies to a requirements.txt file.
 
+    Args:
+        ctx: The context instance (passed automatically).
+
     Example:
         `duty export`
     """
@@ -142,11 +190,12 @@ def export(ctx):
 
 
 @duty
-def publish(ctx, password:str):
+def publish(ctx, password: str):
     """
     Publish the package to pypi.org.
 
     Args:
+        ctx: The context instance (passed automatically).
         password (str): pypi.org password.
 
     Example:
@@ -160,6 +209,159 @@ def publish(ctx, password:str):
     print(publish_result)
 
 
+@duty(silent=True)
+def clean(ctx):
+    """
+    Delete temporary files.
+
+    Args:
+        ctx: The context instance (passed automatically).
+    """
+    ctx.run("rm -rf .mypy_cache")
+    ctx.run("rm -rf .pytest_cache")
+    ctx.run("rm -rf tests/.pytest_cache")
+    ctx.run("rm -rf build")
+    ctx.run("rm -rf dist")
+    ctx.run("rm -rf pip-wheel-metadata")
+    ctx.run("rm -rf site")
+    ctx.run("rm -rf coverage.xml")
+    ctx.run("rm -rf pytest.xml")
+    ctx.run("rm -rf htmlcov")
+    ctx.run("find . -iname '.coverage*' -not -name .coveragerc | xargs rm -rf")
+    ctx.run("find . -type d -name __pycache__ | xargs rm -rf")
+    ctx.run("find . -name '*.rej' -delete")
+
+
+@duty
+def format(ctx):
+    """
+    Format code using Black and isort.
+
+    Args:
+        ctx: The context instance (passed automatically).
+    """
+    res = ctx.run(["black", "--line-length=99", PACKAGE_NAME], pty=True, title="Running Black")
+    print(res)
+
+    res = ctx.run(["isort", PACKAGE_NAME])
+    print(res)
+
+
+@duty(pre=["check_code_quality", "check_types", "check_docs", "check_dependencies"])
+def check(ctx):
+    """
+    Check the code quality, check types, check documentation builds and check dependencies for vulnerabilities.
+
+    Args:
+        ctx: The context instance (passed automatically).
+    """
+
+
+@duty
+def check_code_quality(ctx):
+    """
+    Check the code quality using prospector.
+
+    Args:
+        ctx: The context instance (passed automatically).
+    """
+    ctx.run(["prospector", PACKAGE_NAME], pty=True, title="Checking code quality with prospector")
+
+
+@duty
+def check_types(ctx):
+    """
+    Check the types using mypy.
+
+    Args:
+        ctx: The context instance (passed automatically).
+    """
+    ctx.run(["mypy", PACKAGE_NAME], pty=True, title="Checking types with MyPy")
+
+
+@duty
+def check_docs(ctx):
+    """
+    Check the documentation builds successfully.
+
+    Args:
+        ctx: The context instance (passed automatically).
+    """
+    ctx.run(["mkdocs", "build"], title="Building documentation")
+
+
+@duty
+def check_dependencies(ctx):
+    """
+    Check dependencies with safety for vulnerabilities.
+
+    Args:
+        ctx: The context instance (passed automatically).
+    """
+    for module in sys.modules:
+        if module.startswith("safety.") or module == "safety":
+            del sys.modules[module]
+
+    importlib.invalidate_caches()
+
+    from safety import safety
+    from safety.formatter import report
+    from safety.util import read_requirements
+
+    requirements = ctx.run(
+        "poetry export --dev --without-hashes",
+        title="Exporting dependencies as requirements",
+        allow_overrides=False,
+    )
+
+    def check_vulns():
+        packages = list(read_requirements(StringIO(requirements)))
+        vulns = safety.check(packages=packages, ignore_ids="41002", key="", db_mirror="", cached=False, proxy={})
+        output_report = report(vulns=vulns, full=True, checked_packages=len(packages))
+        print(vulns)
+        if vulns:
+            print(output_report)
+
+    ctx.run(
+        check_vulns,
+        stdin=requirements,
+        title="Checking dependencies",
+        pty=True,
+    )
+
+
+@duty
+def changelog(ctx, planned_release: Optional[str] = None, previous_release: Optional[str] = None):
+    """
+    Generate a changelog with git-cliff.
+
+    Args:
+        ctx: The context instance (passed automatically).
+        planned_release (str, optional): The planned release version. Example: v1.0.2
+        previous_release (str, optional): The previous release version. Example: v1.0.1
+    """
+    generated_changelog: str = ctx.run(["git", "cliff", "-u", "-t", planned_release, "-s", "header"])[:-1]
+    if previous_release is not None:
+        generated_changelog: list = generated_changelog.splitlines()
+        generated_changelog.insert(
+            1,
+            f"<small>[Compare with {previous_release}]({REPO_URL}/compare/{previous_release}...{planned_release})</small>",
+        )
+        generated_changelog: str = "\n".join([line for line in generated_changelog]) + "\n"
+    new_changelog = []
+
+    changelog_file = pathlib.Path(".") / "CHANGELOG.md"
+    with changelog_file.open("r", encoding="utf-8") as changelog_contents:
+        all_lines = changelog_contents.readlines()
+        for line_string in all_lines:
+            regex_string = re.search(r"(<!-- marker -->)", line_string)
+            new_changelog.append(line_string)
+            if isinstance(regex_string, re.Match):
+                new_changelog.append(generated_changelog)
+    with changelog_file.open("w", encoding="utf-8") as changelog_contents:
+        changelog_contents.writelines(new_changelog)
+
+
 def rm_tree(directory: pathlib.Path):
     """
     Recursively delete a directory and all its contents.
@@ -167,7 +369,7 @@ def rm_tree(directory: pathlib.Path):
     Args:
         directory (pathlib.Path): The directory to delete.
     """
-    for child in directory.glob('*'):
+    for child in directory.glob("*"):
         if child.is_file():
             child.unlink()
         else:

panaetius/_version.py

@@ -1,3 +1,3 @@
 """Module containing the version of panaetius."""
 
-__version__ = "2.3.2"
+__version__ = "2.3.5"

panaetius/config.py

@@ -79,9 +79,7 @@ class Config:
         if self.skip_header_init:
             config_file_location = self.config_path / "config.yml"
         else:
-            config_file_location = (
-                self.config_path / self.header_variable / "config.yml"
-            )
+            config_file_location = self.config_path / self.header_variable / "config.yml"
         try:
             with open(config_file_location, "r", encoding="utf-8") as config_file:
                 # return dict(toml.load(config_file))
@@ -149,15 +147,16 @@ class Config:
             if value is not None:
                 return self.__get_config_value_env_var_override(value)
 
-            if len(key.split(".")) > 2:
+            if len(key.split(".")) > 3:
                 raise KeyErrorTooDeepException(
-                    f"Your key of {key} can only be 2 levels deep maximum. "
-                    f"You have {len(key.split('.'))}"
+                    f"Your key of {key} can only be 3 levels deep maximum."
                 )
             if len(key.split(".")) == 1:
                 return self.__get_config_value_key_split_once(key)
             if len(key.split(".")) == 2:
                 return self.__get_config_value_key_split_twice(key)
+            if len(key.split(".")) == 3:
+                return self.__get_config_value_key_split_thrice(key)
             raise KeyError()
 
         except (KeyError, TypeError):
@@ -174,6 +173,10 @@ class Config:
         section, name = key.lower().split(".")
         return self.config[self.header_variable][section][name]
 
+    def __get_config_value_key_split_thrice(self, key: str) -> Any:
+        section, name_0, name_1 = key.lower().split(".")
+        return self.config[self.header_variable][section][name_0][name_1]
+
     def __get_config_value_missing_key_value_is_none(self, default: Any) -> Any:
         return self.__load_default_value(default)
 

panaetius/logging.py

@@ -106,6 +106,11 @@ class LoggingData(metaclass=ABCMeta):
     def format(self) -> str:
         raise NotImplementedError
 
+    @property
+    @abstractmethod
+    def logging_level(self) -> str:
+        raise NotImplementedError
+
     @abstractmethod
     def __init__(self, logging_level: str):
         raise NotImplementedError
@@ -119,8 +124,12 @@ class SimpleLogger(LoggingData):
             '"%(levelname)s",\n\t"message": "%(message)s"\n}',
         )
 
+    @property
+    def logging_level(self) -> str:
+        return self._logging_level
+
     def __init__(self, logging_level: str = "INFO"):
-        self.logging_level = logging_level
+        self._logging_level = logging_level
 
 
 class AdvancedLogger(LoggingData):
@@ -133,8 +142,12 @@ class AdvancedLogger(LoggingData):
             '"%(levelname)s",\n\t"message": "%(message)s"\n}',
         )
 
+    @property
+    def logging_level(self) -> str:
+        return self._logging_level
+
     def __init__(self, logging_level: str = "INFO"):
-        self.logging_level = logging_level
+        self._logging_level = logging_level
 
 
 class CustomLogger(LoggingData):
@@ -142,6 +155,10 @@ class CustomLogger(LoggingData):
     def format(self) -> str:
         return str(self._format)
 
+    @property
+    def logging_level(self) -> str:
+        return self._logging_level
+
     def __init__(self, logging_format: str, logging_level: str = "INFO"):
-        self.logging_level = logging_level
+        self._logging_level = logging_level
         self._format = logging_format

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "panaetius"
-version = "2.3.2"
+version = "2.3.5"
 description = "Python module to gracefully handle a .config file/environment variables for scripts, with built in masking for sensitive options. Provides a Splunk friendly formatted logger instance."
 license = "MIT"
 authors = ["dtomlinson <dtomlinson@panaetius.co.uk>"]
@@ -24,17 +24,32 @@ classifiers = [
 
 [tool.poetry.dependencies]
 python = "^3.7"
-toml = "^0.10.0"
-PyYAML = "^6.0"
+PyYAML = "*"
 
 [tool.poetry.dev-dependencies]
 prospector = {extras = ["with_bandit", "with_mypy"], version = "^1.5.1"}
-types-toml = "^0.10.1"
 pytest = "^6.2.5"
 pytest-datadir = "^1.3.1"
 pytest-xdist = "^2.4.0"
 coverage = "^6.0.2"
 duty = "^0.7.0"
+types-PyYAML = "*"
+isort = "^5.10.1"
+mypy = "^0.910"
+safety = "^1.10.3"
+
+[tool.black]
+line-length = 120
+
+[tool.isort]
+line-length = 120
+not_skip = "__init__.py"
+multi_line_output = 3
+force_single_line = false
+balanced_wrapping = true
+default_section = "THIRDPARTY"
+known_first_party = "duty"
+include_trailing_comma = true
 
 [build-system]
 requires = ["poetry>=0.12"]

requirements.txt

@@ -1,2 +1 @@
 pyyaml==6.0; python_version >= "3.6"
-toml==0.10.2; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.3.0")

requirements_dev.txt

@@ -1,60 +1,67 @@
 ansimarkup==1.5.0; python_version >= "3.6"
-astroid==2.8.2; python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.6.1"
+astroid==2.9.3; python_full_version >= "3.6.2" and python_version < "4.0"
 atomicwrites==1.4.0; python_version >= "3.6" and python_full_version < "3.0.0" and sys_platform == "win32" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6") or sys_platform == "win32" and python_version >= "3.6" and python_full_version >= "3.4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6")
-attrs==21.2.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
-bandit==1.7.0; python_full_version >= "3.6.1" and python_version < "4.0" and python_version >= "3.5"
+attrs==21.4.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
+bandit==1.7.2; python_full_version >= "3.6.2" and python_version < "4.0" and python_version >= "3.7"
 cached-property==1.5.2; python_version < "3.8" and python_version >= "3.6"
-colorama==0.4.4; sys_platform == "win32" and python_version >= "3.6" and python_full_version >= "3.6.1" and python_version < "4.0" and platform_system == "Windows" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6")
-coverage==6.0.2; python_version >= "3.6"
-dodgy==0.2.1; python_full_version >= "3.6.1" and python_version < "4.0"
+certifi==2021.10.8; python_version >= "3.5" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.5"
+charset-normalizer==2.0.10; python_full_version >= "3.6.0" and python_version >= "3.5"
+click==8.0.3; python_version >= "3.6"
+colorama==0.4.4; sys_platform == "win32" and python_version >= "3.7" and python_full_version >= "3.6.2" and python_version < "4.0" and platform_system == "Windows" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6")
+coverage==6.3; python_version >= "3.7"
+dodgy==0.2.1; python_full_version >= "3.6.2" and python_version < "4.0"
+dparse==0.5.1; python_version >= "3.5"
 duty==0.7.0; python_version >= "3.6"
 execnet==1.9.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
 failprint==0.8.0; python_version >= "3.6"
-flake8-polyfill==1.0.2; python_full_version >= "3.6.1" and python_version < "4.0"
-flake8==2.3.0; python_full_version >= "3.6.1" and python_version < "4.0"
-gitdb==4.0.7; python_full_version >= "3.6.1" and python_version < "4.0" and python_version >= "3.7"
-gitpython==3.1.24; python_full_version >= "3.6.1" and python_version < "4.0" and python_version >= "3.7"
-importlib-metadata==4.8.1; python_version < "3.8" and python_version >= "3.6" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6") and python_full_version >= "3.6.1"
+flake8-polyfill==1.0.2; python_full_version >= "3.6.2" and python_version < "4.0"
+flake8==2.3.0; python_full_version >= "3.6.2" and python_version < "4.0"
+gitdb==4.0.9; python_full_version >= "3.6.2" and python_version < "4.0" and python_version >= "3.7"
+gitpython==3.1.26; python_full_version >= "3.6.2" and python_version < "4.0" and python_version >= "3.7"
+idna==3.3; python_version >= "3.5" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.5"
+importlib-metadata==4.10.1; python_version < "3.8" and python_version >= "3.7" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6") and python_full_version >= "3.6.2"
 iniconfig==1.1.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
-isort==5.9.3; python_full_version >= "3.6.1" and python_version < "4.0" and python_version >= "3.6"
+isort==5.10.1; python_full_version >= "3.6.1" and python_version < "4.0"
 jinja2==3.0.3; python_version >= "3.6"
-lazy-object-proxy==1.6.0; python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.6.1"
+lazy-object-proxy==1.7.1; python_full_version >= "3.6.2" and python_version < "4.0" and python_version >= "3.6"
 markupsafe==2.0.1; python_version >= "3.6"
-mccabe==0.6.1; python_full_version >= "3.6.1" and python_version < "4.0" and python_version >= "3.6"
-mypy-extensions==0.4.3; python_full_version >= "3.6.1" and python_version < "4.0" and python_version >= "3.5"
-mypy==0.910; python_full_version >= "3.6.1" and python_version < "4.0" and python_version >= "3.5"
-packaging==21.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
-pbr==5.6.0; python_full_version >= "3.6.1" and python_version < "4.0" and python_version >= "3.6"
-pep8-naming==0.10.0; python_full_version >= "3.6.1" and python_version < "4.0"
-pep8==1.7.1; python_full_version >= "3.6.1" and python_version < "4.0"
-platformdirs==2.4.0; python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.6.1"
+mccabe==0.6.1; python_full_version >= "3.6.2" and python_version < "4.0"
+mypy-extensions==0.4.3; python_full_version >= "3.6.2" and python_version < "4.0" and python_version >= "3.5"
+mypy==0.910; python_version >= "3.5"
+packaging==21.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
+pbr==5.8.0; python_full_version >= "3.6.2" and python_version < "4.0" and python_version >= "3.7"
+pep8-naming==0.10.0; python_full_version >= "3.6.2" and python_version < "4.0"
+pep8==1.7.1; python_full_version >= "3.6.2" and python_version < "4.0"
+platformdirs==2.4.1; python_full_version >= "3.6.2" and python_version < "4.0" and python_version >= "3.7"
 pluggy==1.0.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
-prospector==1.5.1; python_full_version >= "3.6.1" and python_version < "4.0"
+prospector==1.6.0; python_full_version >= "3.6.2" and python_version < "4.0"
 ptyprocess==0.7.0; sys_platform != "win32" and python_version >= "3.6"
-py==1.10.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
-pycodestyle==2.8.0; python_full_version >= "3.6.1" and python_version < "4.0"
-pydocstyle==6.1.1; python_full_version >= "3.6.1" and python_version < "4.0" and python_version >= "3.6"
-pyflakes==2.3.1; python_full_version >= "3.6.1" and python_version < "4.0"
-pylint-celery==0.3; python_full_version >= "3.6.1" and python_version < "4.0"
-pylint-django==2.4.4; python_full_version >= "3.6.1" and python_version < "4.0"
-pylint-flask==0.6; python_full_version >= "3.6.1" and python_version < "4.0"
-pylint-plugin-utils==0.6; python_full_version >= "3.6.1" and python_version < "4.0"
-pylint==2.11.1; python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.6.1"
-pyparsing==2.4.7; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6"
+py==1.11.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
+pycodestyle==2.8.0; python_full_version >= "3.6.2" and python_version < "4.0"
+pydocstyle==6.1.1; python_full_version >= "3.6.2" and python_version < "4.0" and python_version >= "3.6"
+pyflakes==2.3.1; python_full_version >= "3.6.2" and python_version < "4.0"
+pylint-celery==0.3; python_full_version >= "3.6.2" and python_version < "4.0"
+pylint-django==2.5.0; python_full_version >= "3.6.2" and python_version < "4.0"
+pylint-flask==0.6; python_full_version >= "3.6.2" and python_version < "4.0"
+pylint-plugin-utils==0.7; python_full_version >= "3.6.2" and python_version < "4.0"
+pylint==2.12.2; python_full_version >= "3.6.2" and python_version < "4.0"
+pyparsing==3.0.7; python_version >= "3.6"
 pytest-datadir==1.3.1; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0")
-pytest-forked==1.3.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
-pytest-xdist==2.4.0; python_version >= "3.6"
+pytest-forked==1.4.0; python_version >= "3.6"
+pytest-xdist==2.5.0; python_version >= "3.6"
 pytest==6.2.5; python_version >= "3.6"
 pyyaml==6.0; python_version >= "3.6"
-requirements-detector==0.7; python_full_version >= "3.6.1" and python_version < "4.0"
-setoptconf-tmp==0.3.1; python_full_version >= "3.6.1" and python_version < "4.0"
-six==1.16.0; python_full_version >= "3.6.1" and python_version < "4.0" and python_version >= "3.5"
-smmap==4.0.0; python_full_version >= "3.6.1" and python_version < "4.0" and python_version >= "3.7"
-snowballstemmer==2.1.0; python_full_version >= "3.6.1" and python_version < "4.0" and python_version >= "3.6"
-stevedore==3.4.0; python_full_version >= "3.6.1" and python_version < "4.0" and python_version >= "3.6"
-toml==0.10.2; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.3.0")
-typed-ast==1.4.3; python_full_version >= "3.6.1" and python_version < "3.8" and python_version >= "3.6" and implementation_name == "cpython"
-types-toml==0.10.1
-typing-extensions==3.10.0.2; python_full_version >= "3.6.1" and python_version < "3.8" and python_version >= "3.7"
-wrapt==1.12.1; python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.6.1"
-zipp==3.6.0; python_version < "3.8" and python_version >= "3.6"
+requests==2.27.1; python_version >= "3.5" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.5"
+requirements-detector==0.7; python_full_version >= "3.6.2" and python_version < "4.0"
+safety==1.10.3; python_version >= "3.5"
+setoptconf-tmp==0.3.1; python_full_version >= "3.6.2" and python_version < "4.0"
+smmap==5.0.0; python_full_version >= "3.6.2" and python_version < "4.0" and python_version >= "3.7"
+snowballstemmer==2.2.0; python_full_version >= "3.6.2" and python_version < "4.0" and python_version >= "3.6"
+stevedore==3.5.0; python_full_version >= "3.6.2" and python_version < "4.0" and python_version >= "3.7"
+toml==0.10.2; python_full_version >= "3.6.2" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6") and python_version >= "3.5" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6") and (python_version >= "3.5" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.5")
+typed-ast==1.4.3; python_version < "3.8" and python_version >= "3.5" and python_full_version >= "3.6.2" and implementation_name == "cpython"
+types-pyyaml==6.0.3
+typing-extensions==4.0.1; python_full_version >= "3.6.2" and python_version < "3.8" and python_version >= "3.7"
+urllib3==1.26.8; python_version >= "3.5" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version < "4" and python_version >= "3.5"
+wrapt==1.13.3; python_full_version >= "3.6.2" and python_version < "4.0"
+zipp==3.7.0; python_version < "3.8" and python_version >= "3.7"

setup.py

@@ -8,13 +8,13 @@ package_data = \
 {'': ['*']}
 
 install_requires = \
-['PyYAML>=6.0,<7.0', 'toml>=0.10.0,<0.11.0']
+['PyYAML']
 
 setup_kwargs = {
     'name': 'panaetius',
-    'version': '2.3.1',
+    'version': '2.3.5',
     'description': 'Python module to gracefully handle a .config file/environment variables for scripts, with built in masking for sensitive options. Provides a Splunk friendly formatted logger instance.',
-    'long_description': '# Panaetius\n\nThis package provides:\n\n- Functionality to read user variables from a `config.yml` or environment variables.\n- A convenient default logging formatter printing `json` that can save to disk and rotate.\n- Utility functions.\n\n## Config\n\n### options\n\n#### skip_header_init\n\nIf `skip_header_init=True` then the `config_path` will not use the `header_variable` as the\nsub-directory in the `config_path`.\n\nE.g\n\n`CONFIG = panaetius.Config("tembo", "~/tembo/.config", skip_header_init=True)`\n\nWill look in `~/tembo/config/config.yml`.\n\nIf `skip_header_init=False` then would look in `~/tembo/config/tembo/config.yml`.\n\n### Module\n\nConvenient to place in a package/sub-package `__init__.py`.\n\nSee Tembo for an example: <https://github.com/tembo-pages/tembo-core/blob/main/tembo/cli/__init__.py>\n\nExample snippet to use in a module:\n\n```python\nimport os\n\nimport panaetius\nfrom panaetius.exceptions import LoggingDirectoryDoesNotExistException\n\n\nif (config_path := os.environ.get("TEMBO_CONFIG")) is not None:\n    CONFIG = panaetius.Config("tembo", config_path, skip_header_init=True)\nelse:\n    CONFIG = panaetius.Config("tembo", "~/tembo/.config", skip_header_init=True)\n\n\npanaetius.set_config(CONFIG, "base_path", "~/tembo")\npanaetius.set_config(CONFIG, "template_path", "~/tembo/.templates")\npanaetius.set_config(CONFIG, "scopes", {})\npanaetius.set_config(CONFIG, "logging.level", "DEBUG")\npanaetius.set_config(CONFIG, "logging.path")\n\ntry:\n    logger = panaetius.set_logger(\n        CONFIG, panaetius.SimpleLogger(logging_level=CONFIG.logging_level)\n    )\nexcept LoggingDirectoryDoesNotExistException:\n    _LOGGING_PATH = CONFIG.logging_path\n    CONFIG.logging_path = ""\n    logger = panaetius.set_logger(\n        CONFIG, panaetius.SimpleLogger(logging_level=CONFIG.logging_level)\n    )\n    logger.warning("Logging directory %s does not exist", _LOGGING_PATH)\n\n```\n\nThis means in `./tembo/cli/cli.py` you can\n\n```python\nimport tembo.cli\n\n# access the CONFIG instance + variables from the config.yml\ntembo.cli.CONFIG\n```\n\n\n## Utility Functions\n\n### Squasher\n\nSquashes a json object or Python dictionary into a single level dictionary.\n',
+    'long_description': '# Panaetius\n\nThis package provides:\n\n- Functionality to read user variables from a `config.yml` or environment variables.\n- A convenient default logging formatter printing `json` that can save to disk and rotate.\n- Utility functions.\n\n## Config\n\n### options\n\n#### skip_header_init\n\nIf `skip_header_init=True` then the `config_path` will not use the `header_variable` as the\nsub-directory in the `config_path`.\n\nE.g\n\n`CONFIG = panaetius.Config("tembo", "~/tembo/.config", skip_header_init=True)`\n\nWill look in `~/tembo/config/config.yml`.\n\nIf `skip_header_init=False` then would look in `~/tembo/config/tembo/config.yml`.\n\n### Module\n\nConvenient to place in a package/sub-package `__init__.py`.\n\nSee Tembo for an example: <https://github.com/tembo-pages/tembo-core/blob/main/tembo/cli/__init__.py>\n\nExample snippet to use in a module:\n\n```python\n"""Subpackage that contains the CLI application."""\n\nimport os\nfrom typing import Any\n\nimport panaetius\nfrom panaetius.exceptions import LoggingDirectoryDoesNotExistException\n\n\nif (config_path := os.environ.get("TEMBO_CONFIG")) is not None:\n    CONFIG: Any = panaetius.Config("tembo", config_path, skip_header_init=True)\nelse:\n    CONFIG = panaetius.Config(\n        "tembo", "~/tembo/.config", skip_header_init=True\n    )\n\n\npanaetius.set_config(CONFIG, "base_path", "~/tembo")\npanaetius.set_config(CONFIG, "template_path", "~/tembo/.templates")\npanaetius.set_config(CONFIG, "scopes", {})\npanaetius.set_config(CONFIG, "logging.level", "DEBUG")\npanaetius.set_config(CONFIG, "logging.path")\n\ntry:\n    logger = panaetius.set_logger(\n        CONFIG, panaetius.SimpleLogger(logging_level=CONFIG.logging_level)\n    )\nexcept LoggingDirectoryDoesNotExistException:\n    _LOGGING_PATH = CONFIG.logging_path\n    CONFIG.logging_path = ""\n    logger = panaetius.set_logger(\n        CONFIG, panaetius.SimpleLogger(logging_level=CONFIG.logging_level)\n    )\n    logger.warning("Logging directory %s does not exist", _LOGGING_PATH)\n\n```\n\nThis means in `./tembo/cli/cli.py` you can\n\n```python\nimport tembo.cli\n\n# access the CONFIG instance + variables from the config.yml\ntembo.cli.CONFIG\n```\n\n### Script\n\nCreate `./config/config.yml` in the same directory as the script.\n\nIn the script initialise a `CONFIG` object:\n\n```python\nimport pathlib\n\nimport panaetius\n\nCONFIG = panaetius.Config(\n    "teenagers_scraper", str(pathlib.Path(__file__).parents[0] / ".config"), skip_header_init=True\n)\n```\n\nSet variables in the same way as the module above.\n\n#### quickstart logging\n\n```python\nimport panaetius\n\n\ndef get_logger():\n    logging_dir = pathlib.Path(__file__).parents[0] / "logs"\n    logging_dir.mkdir(parents=True, exist_ok=True)\n\n    CONFIG = panaetius.Config("training_data_into_gcp", skip_header_init=True)\n    panaetius.set_config(CONFIG, "logging.level", "DEBUG")\n    panaetius.set_config(CONFIG, "logging.path", logging_dir)\n    return panaetius.set_logger(CONFIG, panaetius.SimpleLogger(logging_level=CONFIG.logging_level))\n```\n\n## Utility Functions\n\n### Squasher\n\nSquashes a json object or Python dictionary into a single level dictionary.\n',
     'author': 'dtomlinson',
     'author_email': 'dtomlinson@panaetius.co.uk',
     'maintainer': None,

tests/conftest.py

@@ -18,6 +18,9 @@ def testing_config_contents():
                 "some_second_list": ["some", "second", "value"],
                 "some_second_table": {"first": ["some", "first", "value"]},
                 "some_second_table_bools": {"bool": [True, False]},
+                "third": {
+                    "some_third_string": "some_third_value",
+                },
             },
         }
     }

tests/data/without_logging/panaetius_testing/config.yml

@@ -7,3 +7,5 @@
     some_second_list: ["some", "second", "value"]
     some_second_table: { "first": ["some", "first", "value"] }
     some_second_table_bools: { "bool": [true, false] }
+    third:
+      some_third_string: some_third_value

tests/test_config.py

@@ -67,9 +67,7 @@ def test_config_file_without_header_dir_exists(header, shared_datadir):
     assert config._missing_config is False
 
 
-def test_config_file_contents_read_success(
-    header, shared_datadir, testing_config_contents
-):
+def test_config_file_contents_read_success(header, shared_datadir, testing_config_contents):
     # arrange
     config_path = str(shared_datadir / "without_logging")
 
@@ -101,11 +99,10 @@ def test_config_file_contents_read_success(
             "second_some_second_table_bools",
             {"bool": [True, False]},
         ),
+        ("second.third.some_third_string", "second_third_some_third_string", "some_third_value"),
     ],
 )
-def test_get_value_from_key(
-    set_config_key, get_config_key, expected_value, header, shared_datadir
-):
+def test_get_value_from_key(set_config_key, get_config_key, expected_value, header, shared_datadir):
     """
     Test the following:
 
@@ -155,11 +152,7 @@ def test_key_level_too_deep(header, shared_datadir):
         panaetius.set_config(config, key)
 
     # assert
-    assert (
-        str(key_error_too_deep.value)
-        == f"Your key of {key} can only be 2 levels deep maximum. "
-        f"You have 4"
-    )
+    assert str(key_error_too_deep.value) == f"Your key of {key} can only be 3 levels deep maximum."
 
 
 def test_get_value_missing_key_from_default(header, shared_datadir):
@@ -242,9 +235,7 @@ def test_missing_config_read_from_default(header, shared_datadir):
         ),
     ],
 )
-def test_missing_config_read_from_env_var(
-    env_value, expected_value, header, shared_datadir
-):
+def test_missing_config_read_from_env_var(env_value, expected_value, header, shared_datadir):
     # arrange
     config_path = str(shared_datadir / str(uuid4()))
     os.environ[f"{header.upper()}_MISSING_KEY_READ_FROM_ENV_VAR"] = env_value
@@ -271,10 +262,7 @@ def test_missing_config_read_from_env_var_invalid_python(header):
         panaetius.set_config(config, "invalid_python")
 
     # assert
-    assert (
-        str(invalid_python_exception.value)
-        == "a string without quotes is not valid Python."
-    )
+    assert str(invalid_python_exception.value) == "a string without quotes is not valid Python."
 
     # cleanup
     del os.environ[f"{header.upper()}_INVALID_PYTHON"]