updating tests

.coveragerc

@@ -0,0 +1,7 @@
+[report]
+exclude_lines =
+    # Have to re-enable the standard pragma
+    pragma: no cover
+
+    # Don't complain if tests don't hit defensive assertion code:
+    raise NotImplementedError

.gitignore

@@ -127,3 +127,6 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
+
+# custom
+.DS_Store

README.md.old

@@ -1,33 +0,0 @@
-# Author
-
-Daniel Tomlinson (dtomlinson@panaetius.co.uk)
-
-# Requires
-
-`>= python3.7`
-
-# Python requirements
-
-- toml = "^0.10.0"
-- pylite = "^0.1.0"
-
-# Documentation
-
-_soon_
-
-# Installation
-
-_soon_
-
-# Easy Way
-
-## Python
-
-### From pip
-
-### From local wheel
-
-### From source
-
-# Example Usage
-

TODO

@@ -1,3 +0,0 @@
-
-Todo:
-  ☐ Item

panaetius/config.py

@@ -15,31 +15,41 @@ from typing import Any
 # import toml
 import yaml
 
-from panaetius.exceptions import KeyErrorTooDeepException, InvalidPythonException
+from panaetius.exceptions import KeyErrorTooDeepException
 
 
 class Config:
     """The configuration class to access variables."""
 
-    def __init__(self, header_variable: str, config_path: str = "") -> None:
+    def __init__(
+        self,
+        header_variable: str,
+        config_path: str | None = None,
+        skip_header_init: bool = False,
+    ) -> None:
         """
         Create a Config object to set and access variables.
 
         Args:
             header_variable (str): Your header variable name.
             config_path (str, optional): The path where the header directory is stored.
-                Defaults to `~/.config`.
+                Defaults to None on initialisation.
+            skip_header_init (bool, optional): If True will not use a header
+                subdirectory in the `config_path`. Defaults to False.
+
+        Examples:
+            `config_path` defaults to None on initialisation but will be set to `~/.config`.
 
-        Example:
             A header of `data_analysis` with a config_path of `~/myapps` will define
                 a config file in `~/myapps/data_analysis/config.yml`.
         """
         self.header_variable = header_variable
         self.config_path = (
-            pathlib.Path(config_path)
-            if config_path
+            pathlib.Path(config_path).expanduser()
+            if config_path is not None
             else pathlib.Path.home() / ".config"
         )
+        self.skip_header_init = skip_header_init
         self._missing_config = self._check_config_file_exists()
 
         # default logging options
@@ -55,7 +65,12 @@ class Config:
         Returns:
             dict: The contents of the `.yml` loaded as a python dictionary.
         """
-        config_file_location = self.config_path / self.header_variable / "config.yml"
+        if self.skip_header_init:
+            config_file_location = self.config_path / "config.yml"
+        else:
+            config_file_location = (
+                self.config_path / self.header_variable / "config.yml"
+            )
         try:
             with open(config_file_location, "r", encoding="utf-8") as config_file:
                 # return dict(toml.load(config_file))
@@ -103,7 +118,10 @@ class Config:
         return self._get_env_value(env_key, default)
 
     def _check_config_file_exists(self) -> bool:
-        config_file_location = self.config_path / self.header_variable / "config.yml"
+        if self.skip_header_init is False:
+            config_file_location = self.config_path / self.header_variable / "config.yml"
+        else:
+            config_file_location = self.config_path / "config.yml"
         try:
             with open(config_file_location, "r", encoding="utf-8"):
                 return False
@@ -165,7 +183,8 @@ class Config:
         try:
             return ast.literal_eval(value)
         except (ValueError, SyntaxError):
-            raise InvalidPythonException(f"{value} is not valid Python.")  # noqa
+            # string without spaces: ValueError, with spaces; SyntaxError
+            return value
 
     def __load_default_value(self, default: Any) -> Any:  # noqa
         return default

panaetius/logging.py

@@ -98,11 +98,11 @@ class LoggingData(metaclass=ABCMeta):
     @property
     @abstractmethod
     def format(self) -> str:
-        pass
+        raise NotImplementedError
 
     @abstractmethod
     def __init__(self, logging_level: str):
-        self.logging_level = logging_level
+        raise NotImplementedError
 
 
 class SimpleLogger(LoggingData):

panaetius/utilities/__init__.py

@@ -0,0 +1,3 @@
+"""General utilities."""
+
+from panaetius.utilities.squasher import Squash

panaetius/utilities/squasher.py

@@ -0,0 +1,64 @@
+"""Squash a json object or Python dictionary into a single level dictionary."""
+
+from __future__ import annotations
+
+from copy import deepcopy
+import itertools
+from typing import Iterator, Tuple
+
+
+class Squash:
+    """Squash a json object or Python dictionary into a single level dictionary."""
+
+    def __init__(self, data: dict) -> None:
+        """
+        Create a Squash object to squash data into a single level dictionary.
+
+        Args:
+            data (dict): [description]
+
+        Example:
+            squashed_data = Squash(my_data)
+
+            squashed_data.as_dict
+        """
+        self.data = data
+
+    @property
+    def as_dict(self) -> dict:
+        """
+        Return the squashed data as a dictionary.
+
+        Returns:
+            dict: The original data squashed as a dict.
+        """
+        return self._squash()
+
+    @staticmethod
+    def _unpack_dict(
+        key: str, value: dict | list | str
+    ) -> Iterator[Tuple[str, dict | list | str]]:
+        if isinstance(value, dict):
+            for sub_key, sub_value in value.items():
+                temporary_key = f"{key}_{sub_key}"
+                yield temporary_key, sub_value
+        elif isinstance(value, list):
+            for index, sub_value in enumerate(value):
+                temporary_key = f"{key}_{index}"
+                yield temporary_key, sub_value
+        else:
+            yield key, value
+
+    def _squash(self) -> dict:
+        result = deepcopy(self.data)
+        while True:
+            result = dict(
+                itertools.chain.from_iterable(
+                    itertools.starmap(self._unpack_dict, result.items())
+                )
+            )
+            if not any(
+                isinstance(value, dict) for value in result.values()
+            ) and not any(isinstance(value, list) for value in result.values()):
+                break
+        return result

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "panaetius"
-version = "1.1"
+version = "2.2.1"
 description = "Python module to gracefully handle a .config file/environment variables for scripts, with built in masking for sensitive options. Provides a Splunk friendly formatted logger instance."
 license = "MIT"
 authors = ["dtomlinson <dtomlinson@panaetius.co.uk>"]

rewrite.todo

@@ -1,16 +1,26 @@
+Testing:
+  To Write:
+    ☐ Test the Config file skipping header with `skip_header_init`
+    ☐ Document coverage commands
+      `coverage run --source=./panaetius -m pytest`
+      `coverage report` & `coverage html` > gives ./htmlcov/index.html
+    ☐ Document for abstract methods should raise NotImplementedError
+    ☐ Document https://stackoverflow.com/a/9212387
 Documentation:
     ☐ Rewrite documentation using `mkdocs` and using `.md`.
     ☐ Update the metadata in the `pyproject.toml`.
     ☐ Create a new `Readme.md` and remove the `.rst`.
+    ☐ Document the logging strategy
+      CLI tools should use `logger.critical` and raise SystemExit(1)
+      Libraries should raise custom errors and have a `logger.critical(exec_info=1)`
 
 Misc:
     ☐ Use the python runner to build the docs & run the tests (including coverage html)
         coverage run -m pytest && coverage report && coverage html
         ☐ document this in trilium
-    ☐ Bump the version to release 2.0
-
 
 Archive:
+  ✘ Bump the version to release 2.0 @cancelled(21-10-23 05:36) @project(Misc)
   ✔ Handle if a bool is passed in as a default @done(21-10-16 05:25) @project(Coding.No Config File)
   ✔ Handle if a bool is passed in as a default @done(21-10-16 05:25) @project(Coding.Config File)
   ✔ Create SimpleLogger, AdvancedLogger, CustomLogger classes @done(21-10-16 16:22) @project(Coding.Logging)

setup.py

@@ -1,27 +1,25 @@
 # -*- coding: utf-8 -*-
-from distutils.core import setup
-
-package_dir = \
-{'': 'src'}
+from setuptools import setup
 
 packages = \
-['panaetius']
+['panaetius', 'panaetius.utilities']
 
 package_data = \
 {'': ['*']}
 
 install_requires = \
-['pylite>=0.1.0,<0.2.0', 'toml>=0.10.0,<0.11.0']
+['PyYAML>=6.0,<7.0', 'toml>=0.10.0,<0.11.0']
 
 setup_kwargs = {
     'name': 'panaetius',
-    'version': '1.0.2',
-    'description': 'Python module to gracefully handle a .config file/environment variables for scripts, with built in masking for sensitive options. Provides a Splunk friendly logger instance.',
-    'long_description': 'Author\n=======\n\nDaniel Tomlinson (dtomlinson@panaetius.co.uk)\n\nRequires\n=========\n\n`>= python3.7`\n\nPython requirements\n====================\n\n- toml = "^0.10.0"\n- pylite = "^0.1.0"\n\nDocumentation\n==============\n\nRead the documentation on `read the docs`_.\n\n.. _read the docs: https://panaetius.readthedocs.io/en/latest/introduction.html\n\nInstallation\n==============\n\nYou can install ..:obj:`panaetius`\n\nEasy Way\n=========\n\nPython\n-------\n\nFrom pip\n~~~~~~~~~\n\nFrom local wheel\n~~~~~~~~~~~~~~~~~\n\nFrom source\n~~~~~~~~~~~~\n\nExample Usage\n==============\n\n',
+    'version': '2.2.1',
+    'description': 'Python module to gracefully handle a .config file/environment variables for scripts, with built in masking for sensitive options. Provides a Splunk friendly formatted logger instance.',
+    'long_description': 'Author\n=======\n\nDaniel Tomlinson (dtomlinson@panaetius.co.uk)\n\nRequires\n=========\n\n`>= python3.7`\n\nPython requirements\n====================\n\n- toml = "^0.10.0"\n- pylite = "^0.1.0"\n\nDocumentation\n==============\n\nRead the documentation on `read the docs`_.\n\n.. _read the docs: https://panaetius.readthedocs.io/en/latest/introduction.html\n\nInstallation\n==============\n\nYou can install ``panaetius`` the following ways:\n\nPython\n-------\n\n.. Attention:: You should install in a python virtual environment\n\nFrom pypi using pip\n~~~~~~~~~~~~~~~~~~~~\n\n.. code-block:: bash\n\n    pip install panaetius\n\nFrom local wheel\n~~~~~~~~~~~~~~~~~\n\nDownload the latest verion from the `releases`_ page.\n\n.. _releases: https://github.com/dtomlinson91/panaetius/releases\n\nInstall with pip:\n\n.. code-block:: bash\n\n    pip install -U panaetius-1.0.2-py3-none-any.whl\n\n\nFrom source\n~~~~~~~~~~~~\n\nClone the repo and install using ``setup.py``:\n\n.. code-block:: bash\n\n    python setup.py\n',
     'author': 'dtomlinson',
     'author_email': 'dtomlinson@panaetius.co.uk',
+    'maintainer': None,
+    'maintainer_email': None,
     'url': 'https://github.com/dtomlinson91/panaetius',
-    'package_dir': package_dir,
     'packages': packages,
     'package_data': package_data,
     'install_requires': install_requires,

tests/data/without_header/config.yml

@@ -0,0 +1,9 @@
+panaetius_testing:
+  some_top_string: some_top_value
+  second:
+    some_second_string: some_second_value
+    some_second_int: 1
+    some_second_float: 1.0
+    some_second_list: ["some", "second", "value"]
+    some_second_table: { "first": ["some", "first", "value"] }
+    some_second_table_bools: { "bool": [true, false] }

tests/test_config.py

@@ -29,6 +29,17 @@ def test_user_config_path_set(header, shared_datadir):
     assert str(config.config_path) == config_path
 
 
+def test_user_config_path_without_header_dir_set(header, shared_datadir):
+    # arrange
+    config_path = str(shared_datadir / "without_header")
+
+    # act
+    config = panaetius.Config(header, config_path, skip_header_init=True)
+
+    # assert
+    assert str(config.config_path) == config_path
+
+
 # test config files
 
 
@@ -44,6 +55,18 @@ def test_config_file_exists(header, shared_datadir):
     assert config._missing_config is False
 
 
+def test_config_file_without_header_dir_exists(header, shared_datadir):
+    # arrange
+    config_path = str(shared_datadir / "without_header")
+
+    # act
+    config = panaetius.Config(header, config_path, skip_header_init=True)
+    _ = config.config
+
+    # assert
+    assert config._missing_config is False
+
+
 def test_config_file_contents_read_success(
     header, shared_datadir, testing_config_contents
 ):
@@ -106,7 +129,7 @@ def test_get_value_from_key(
 
 def test_get_value_environment_var_override(header, shared_datadir):
     # arrange
-    os.environ[f"{header.upper()}_SOME_TOP_STRING"] = '"some_overridden_value"'
+    os.environ[f"{header.upper()}_SOME_TOP_STRING"] = "some_overridden_value"
     config_path = str(shared_datadir / "without_logging")
     config = panaetius.Config(header, config_path)
     panaetius.set_config(config, "some_top_string")
@@ -158,7 +181,7 @@ def test_get_value_missing_key_from_default(header, shared_datadir):
 
 def test_get_value_missing_key_from_env(header, shared_datadir):
     # arrange
-    os.environ[f"{header.upper()}_MISSING_KEY"] = '"some missing key"'
+    os.environ[f"{header.upper()}_MISSING_KEY"] = "some missing key"
 
     config_path = str(shared_datadir / "without_logging")
     config = panaetius.Config(header, config_path)
@@ -205,7 +228,7 @@ def test_missing_config_read_from_default(header, shared_datadir):
 @pytest.mark.parametrize(
     "env_value,expected_value",
     [
-        ('"a missing string"', "a missing string"),
+        ("a missing string", "a missing string"),
         ("1", 1),
         ("1.0", 1.0),
         ("True", True),
@@ -237,6 +260,7 @@ def test_missing_config_read_from_env_var(
     del os.environ[f"{header.upper()}_MISSING_KEY_READ_FROM_ENV_VAR"]
 
 
+@pytest.mark.skip(reason="No longer needed as strings are loaded without quotes")
 def test_missing_config_read_from_env_var_invalid_python(header):
     # arrange
     os.environ[f"{header.upper()}_INVALID_PYTHON"] = "a string without quotes"

tests/test_logging.py

@@ -21,6 +21,7 @@ def test_logging_directory_does_not_exist(header, shared_datadir):
     assert str(logging_exception.value) == ""
 
 
+# TODO: change this test so it asserts the dir exists
 def test_logging_directory_does_exist(header, shared_datadir):
     # arrange
     config = Config(header)
@@ -32,3 +33,5 @@ def test_logging_directory_does_exist(header, shared_datadir):
 
     # assert
     assert isinstance(logger, logging.Logger)
+
+# TODO: add tests to check that SimpleLogger, AdvancedLogger, CustomLogger work as intended

tests/test_utilities/test_squasher.py

@@ -0,0 +1,119 @@
+import pytest
+
+from panaetius import utilities
+
+
+def test_squashed_data(squashed_data, squashed_data_result):
+    # act
+    squashed_data_pre_squashed = utilities.squasher.Squash(squashed_data).as_dict
+
+    # assert
+    assert squashed_data_pre_squashed == squashed_data_result
+
+
+@pytest.fixture
+def squashed_data():
+    return {
+        "destination_addresses": [
+            "Washington, DC, USA",
+            "Philadelphia, PA, USA",
+            "Santa Barbara, CA, USA",
+            "Miami, FL, USA",
+            "Austin, TX, USA",
+            "Napa County, CA, USA",
+        ],
+        "origin_addresses": ["New York, NY, USA"],
+        "rows": [
+            {
+                "elements": [
+                    {
+                        "distance": {"text": "227 mi", "value": 365468},
+                        "duration": {
+                            "text": "3 hours 54 mins",
+                            "value": 14064,
+                        },
+                        "status": "OK",
+                    },
+                    {
+                        "distance": {"text": "94.6 mi", "value": 152193},
+                        "duration": {"text": "1 hour 44 mins", "value": 6227},
+                        "status": "OK",
+                    },
+                    {
+                        "distance": {"text": "2,878 mi", "value": 4632197},
+                        "duration": {
+                            "text": "1 day 18 hours",
+                            "value": 151772,
+                        },
+                        "status": "OK",
+                    },
+                    {
+                        "distance": {"text": "1,286 mi", "value": 2069031},
+                        "duration": {
+                            "text": "18 hours 43 mins",
+                            "value": 67405,
+                        },
+                        "status": "OK",
+                    },
+                    {
+                        "distance": {"text": "1,742 mi", "value": 2802972},
+                        "duration": {"text": "1 day 2 hours", "value": 93070},
+                        "status": "OK",
+                    },
+                    {
+                        "distance": {"text": "2,871 mi", "value": 4620514},
+                        "duration": {
+                            "text": "1 day 18 hours",
+                            "value": 152913,
+                        },
+                        "status": "OK",
+                    },
+                ]
+            }
+        ],
+        "status": "OK",
+    }
+
+
+@pytest.fixture
+def squashed_data_result():
+    return {
+        "destination_addresses_0": "Washington, DC, USA",
+        "destination_addresses_1": "Philadelphia, PA, USA",
+        "destination_addresses_2": "Santa Barbara, CA, USA",
+        "destination_addresses_3": "Miami, FL, USA",
+        "destination_addresses_4": "Austin, TX, USA",
+        "destination_addresses_5": "Napa County, CA, USA",
+        "origin_addresses_0": "New York, NY, USA",
+        "rows_0_elements_0_distance_text": "227 mi",
+        "rows_0_elements_0_distance_value": 365468,
+        "rows_0_elements_0_duration_text": "3 hours 54 mins",
+        "rows_0_elements_0_duration_value": 14064,
+        "rows_0_elements_0_status": "OK",
+        "rows_0_elements_1_distance_text": "94.6 mi",
+        "rows_0_elements_1_distance_value": 152193,
+        "rows_0_elements_1_duration_text": "1 hour 44 mins",
+        "rows_0_elements_1_duration_value": 6227,
+        "rows_0_elements_1_status": "OK",
+        "rows_0_elements_2_distance_text": "2,878 mi",
+        "rows_0_elements_2_distance_value": 4632197,
+        "rows_0_elements_2_duration_text": "1 day 18 hours",
+        "rows_0_elements_2_duration_value": 151772,
+        "rows_0_elements_2_status": "OK",
+        "rows_0_elements_3_distance_text": "1,286 mi",
+        "rows_0_elements_3_distance_value": 2069031,
+        "rows_0_elements_3_duration_text": "18 hours 43 mins",
+        "rows_0_elements_3_duration_value": 67405,
+        "rows_0_elements_3_status": "OK",
+        "rows_0_elements_4_distance_text": "1,742 mi",
+        "rows_0_elements_4_distance_value": 2802972,
+        "rows_0_elements_4_duration_text": "1 day 2 hours",
+        "rows_0_elements_4_duration_value": 93070,
+        "rows_0_elements_4_status": "OK",
+        "rows_0_elements_5_distance_text": "2,871 mi",
+        "rows_0_elements_5_distance_value": 4620514,
+        "rows_0_elements_5_duration_text": "1 day 18 hours",
+        "rows_0_elements_5_duration_value": 152913,
+        "rows_0_elements_5_status": "OK",
+        "status": "OK",
+    }