diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000..c88a062 Binary files /dev/null and b/.DS_Store differ diff --git a/README.md b/README.md.old similarity index 100% rename from README.md rename to README.md.old diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..f61f890 --- /dev/null +++ b/README.rst @@ -0,0 +1,61 @@ +Author +======= + +Daniel Tomlinson (dtomlinson@panaetius.co.uk) + +Requires +========= + +`>= python3.7` + +Python requirements +==================== + +- toml = "^0.10.0" +- pylite = "^0.1.0" + +Documentation +============== + +Read the documentation on `read the docs`_. + +.. _read the docs: https://panaetius.readthedocs.io/en/latest/introduction.html + +Installation +============== + +You can install ``panaetius`` the following ways: + +Python +------- + +Install into a virtual environment. + +From pypi using pip +~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: bash + + pip install panaetius + +From local wheel +~~~~~~~~~~~~~~~~~ + +Download the latest verion from the `releases`_ page. + +.. _releases: https://github.com/dtomlinson91/panaetius/releases + +Install with pip: + +.. code-block:: bash + pip install -U panaetius-1.0.2-py3-none-any.whl + + +From source +~~~~~~~~~~~~ + +Clone the repo and install using ``setup.py``: + +.. code-block:: bash + + python setup.py diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst index 9f3d977..80a4995 100644 --- a/docs/source/changelog.rst +++ b/docs/source/changelog.rst @@ -1,7 +1,12 @@ Version history ================ +1.0.2 +------ + +- Minor fixes and documentation updates. + 1.0 -------- -- initial release +- Initial release. diff --git a/docs/source/configuration.rst b/docs/source/configuration.rst index 972dd83..b9f0054 100644 --- a/docs/source/configuration.rst +++ b/docs/source/configuration.rst @@ -1,4 +1,23 @@ Configuration ============= -.. todo:: fill in configuration options +panaetius is fairly easy to configure. There are just a couple of options to be aware of. + +header.py +---------- + +You should set a ``__header__.py`` next to your script or module. + +This ``__header__.py`` should contain a ``__header__`` variable that sets the name of your project/script. + +E.g a ``__header__.py`` for the module ``plex_posters`` would look like: + +.. code-block:: python + + __header__ = 'plex_posters' + +If you are writing a script, simply place this ``__header__.py`` along side your script. Panaetius will pick this up when the script is ran. + +If you are writing a module, you can either place the ``__header__.py`` alongside the script that uses your module. If this is not possible, panaetius will set the default ``__header__`` variable to the name of the virtualenv that the script is activated from. + +If neither of the above aren't possible (say your script is running in a lambda on AWS), then ``__header__`` will be set to the default of ``panaetius``. diff --git a/docs/source/introduction.rst b/docs/source/introduction.rst index 72d6427..9be558d 100644 --- a/docs/source/introduction.rst +++ b/docs/source/introduction.rst @@ -1,4 +1,154 @@ panaetius ========== -tbc +.. image:: https://img.shields.io/readthedocs/panaetius?style=for-the-badge :target: https://panaetius.readthedocs.io/en/latest/?badge=latest + :alt: Documentation Status + +.. image:: https://img.shields.io/github/v/tag/dtomlinson91/panaetius?style=for-the-badge :alt: GitHub tag (latest by date) + +.. image:: https://img.shields.io/github/commit-activity/m/dtomlinson91/panaetius?style=for-the-badge :alt: GitHub commit activity + +.. image:: https://img.shields.io/github/issues/dtomlinson91/panaetius?style=for-the-badge :alt: GitHub issues + +.. image:: https://img.shields.io/github/license/dtomlinson91/panaetius?style=for-the-badge :alt: GitHubtbc + +Python module to gracefully handle a .config file/environment variables for scripts, with built in masking for sensitive options. Provides a Splunk friendly formatted logger instance. + +Usage +------ + +Setting a config file +~~~~~~~~~~~~~~~~~~~~~~ + +The main functionality of ``panaetius`` is using a config file to store variables. + +Your ``config.toml`` can be created and found in ~/.config/__header__/config.toml where __header__ is equal to the variable configured/set. `See how to configure`_ this variable in the configuration section of panaetius. + +.. _See how to configure: https://panaetius.readthedocs.io/en/latest/configuration.html#header-py + +Setting values in a config.toml/environment variables +####################################################### + +A ``config.toml`` can be created in the default folder for the module. In this example this would be found in ``~/.config/example_module/config.toml``. + +An example ``config.toml`` could look like: + +.. code-block:: toml + + [example_module] + test = "a6cbf36649b029f3618a0cc1" + + [example_module.logging] + path = "~/.config/example_module" + level = "DEBUG" + + [example_module.foo] + bar = "6b3b96815218960ceaf7cceb" + +These are equivalent to the environment variables: + +.. code-block:: bash + + EXAMPLE_MODULE_TEST + EXAMPLE_MODULE_LOGGING_PATH + EXAMPLE_MODULE_LOGGING_LEVEL + EXAMPLE_MODULE_FOO_BAR + + +.. Attention:: + Environment variables take precedent over the ``config.toml``. If both are set then the environment variable will be used. + +You can overwrite the ``config.toml`` location by setting the environment variable: + +.. code-block:: bash + + DEFAULT_CONFIG_PATH = "~/path/to/config" + + +Setting values in your code +############################ + +Values in a ``config.toml`` or from an environment variable need to be set in your work in order for you to use them. You can do this easily by + +- importing panaetius. +- using the :func:`panaetius.library.set_config` function. + +E.g your script could contain: + +.. code-block:: python + + import panaetius + panaetius.set_config(panaetius.CONFIG, 'logging.path') + +.. Note:: + + The ``key`` attribute in :func:`panaetius.library.set_config` is specified as a string, with the hirearchy in the config file split with a ``.``. + +.. Important:: + + The default value for a variable defined using :func:`panaetius.library.set_config` is ``None``. See the documentation of this function to see all the options available. + + +Accessing values +################# + +You can then access the result of this variable later in your code: + +.. code-block:: python + + panaetius.CONFIG.logging_path + + +Logging +~~~~~~~~ + +In order to save to disk, you need to specify a path for the log file in the config file/environment variable. There is no need to register this with :func:`panaetius.library.set_config` as ``panaetius`` will do this automatically. + +There are other options available for you to configure a logger. These are (including the default values which can be overwritten): + +.. code-block:: toml + + [example_module.logging] + backup_count = 3 + format = "{\n\t"time": "%(asctime)s",\n\t"file_name": "%(filename)s",' + '\n\t"module": "%(module)s",\n\t"function":"%(funcName)s",\n\t' + '"line_number": "%(lineno)s",\n\t"logging_level":' + '"%(levelname)s",\n\t"message": "%(message)s"\n}" + level = "INFO" # Level should be in CAPS + rotate_bytes = 512000 + +You can use the logger in your code by: + +.. code-block:: python + + panaetius.logger.info('some log message') + +which gives an output of: + +.. code-block:: json + + { + "time": "2020-01-13 23:07:17,913", + "file_name": "test.py", + "module": "test", + "function":"", + "line_number": "33", + "logging_level":"INFO", + "message": "some logging message" + } + + +Importing and using the api +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +See `panaetius api page`_ on how to use and import the module. + +.. _panaetius api page: https://panaetius.readthedocs.io/en/latest/modules/panaetius.html + + +Configuration +--------------- + +See `configuration page`_ on how to configure ``panaetius``. + +.. _configuration page: https://panaetius.readthedocs.io/en/latest/configuration.html diff --git a/docs/source/modules/modules.rst b/docs/source/modules/modules.rst deleted file mode 100644 index 936cb1b..0000000 --- a/docs/source/modules/modules.rst +++ /dev/null @@ -1,7 +0,0 @@ -panaetius -========= - -.. toctree:: - :maxdepth: 4 - - panaetius diff --git a/docs/source/modules/panaetius.config.rst b/docs/source/modules/panaetius.config.rst index 7fcfe0e..285cc5d 100644 --- a/docs/source/modules/panaetius.config.rst +++ b/docs/source/modules/panaetius.config.rst @@ -1,7 +1,7 @@ .. include:: ../global.rst -Config :modname:`panaetius.config` -------------------------------------- +panaetius.config :modname:`panaetius.config` +--------------------------------------------- .. automodule:: panaetius.config :members: diff --git a/docs/source/modules/panaetius.config_inst.rst b/docs/source/modules/panaetius.config_inst.rst index 9ad52be..984af9e 100644 --- a/docs/source/modules/panaetius.config_inst.rst +++ b/docs/source/modules/panaetius.config_inst.rst @@ -1,7 +1,7 @@ .. include:: ../global.rst -Config :modname:`panaetius.config_inst` ---------------------------------------- +panaetius.config_inst :modname:`panaetius.config_inst` +-------------------------------------------------------- .. automodule:: panaetius.config_inst :members: diff --git a/docs/source/modules/panaetius.db.rst b/docs/source/modules/panaetius.db.rst index ab00b8e..6ef6d90 100644 --- a/docs/source/modules/panaetius.db.rst +++ b/docs/source/modules/panaetius.db.rst @@ -1,6 +1,6 @@ .. include:: ../global.rst -Config :modname:`panaetius.db` +panaetius.db :modname:`panaetius.db` ------------------------------------- .. automodule:: panaetius.db diff --git a/docs/source/modules/panaetius.header.rst b/docs/source/modules/panaetius.header.rst index 69ef1a4..f4bb770 100644 --- a/docs/source/modules/panaetius.header.rst +++ b/docs/source/modules/panaetius.header.rst @@ -1,7 +1,7 @@ .. include:: ../global.rst -Config :modname:`panaetius.header` -------------------------------------- +panaetius.header :modname:`panaetius.header` +--------------------------------------------- .. automodule:: panaetius.header :members: diff --git a/docs/source/modules/panaetius.library.rst b/docs/source/modules/panaetius.library.rst index bde30e9..7904df0 100644 --- a/docs/source/modules/panaetius.library.rst +++ b/docs/source/modules/panaetius.library.rst @@ -1,7 +1,7 @@ .. include:: ../global.rst -Config :modname:`panaetius.library` -------------------------------------- +panaetius.library :modname:`panaetius.library` +------------------------------------------------ .. automodule:: panaetius.library :members: diff --git a/docs/source/modules/panaetius.logging.rst b/docs/source/modules/panaetius.logging.rst index b1947d8..e284181 100644 --- a/docs/source/modules/panaetius.logging.rst +++ b/docs/source/modules/panaetius.logging.rst @@ -1,7 +1,7 @@ .. include:: ../global.rst -Config :modname:`panaetius.logging` -------------------------------------- +panaetius.logging :modname:`panaetius.logging` +---------------------------------------------- .. automodule:: panaetius.logging :members: diff --git a/docs/source/modules/panaetius.rst b/docs/source/modules/panaetius.rst index 790e3c9..2506923 100644 --- a/docs/source/modules/panaetius.rst +++ b/docs/source/modules/panaetius.rst @@ -1,21 +1,66 @@ .. include:: ../global.rst -Config :modname:`panaetius` -------------------------------------- +********* +panaetius +********* -.. automodule:: panaetius - :members: - :undoc-members: - :show-inheritance: +API +=== -Submodules ----------- +The following is availble by importing the module: -.. toctree:: +.. code-block:: python - panaetius.config - panaetius.config_inst - panaetius.db - panaetius.header - panaetius.library - panaetius.logging + import panaetius + + +panaetius.CONFIG +---------------- + +:obj:`panaetius.CONFIG` provides an instance of :class:`panaetius.config.Config` + + +panaetius.set_config() +----------------------- + +Conveniently provides :func:`panaetius.library.set_config` + +Use in your module/script with: + +.. code-block:: python + + panaetius.set_config(panaetius.CONFIG, 'aws.secret_key', str, mask=True) + + +panaetius.CONFIG.aws_secret_key +------------------------------- + +Conveniently provides access to all attributes that have been declared with :func:`panaetius.library.set_config`: + +.. code-block:: python + + my_secret_key = panaetius.CONFIG.aws_secret_key + + +panaetius.logger +----------------- + +:obj:`panaetius.logger` provides a logger instance already formatted with a nice json output. + +.. code-block:: python + + panaetius.logger.info('some logging message') + +This gives a logger output of: + +.. code-block:: json + + { + "time": "2020-01-13 23:07:17,913", + "file_name": "test.py", + "module": "test", + "function":"", + "line_number": "33", + "logging_level":"INFO", + "message": "some logging message" + } diff --git a/docs/source/toc.rst b/docs/source/toc.rst index a2ccebd..f6bea3e 100644 --- a/docs/source/toc.rst +++ b/docs/source/toc.rst @@ -25,4 +25,3 @@ modules/panaetius.header.rst modules/panaetius.library.rst modules/panaetius.logging.rst - modules/panaetius.rst diff --git a/pyproject.toml b/pyproject.toml index 4194704..ee252b9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,13 +1,13 @@ [tool.poetry] name = "panaetius" -version = "1.0" -description = "Python module to gracefully handle a .config file/environment variables for scripts, with built in masking for sensitive options. Provides a Splunk friendly logger instance." +version = "1.0.2" +description = "Python module to gracefully handle a .config file/environment variables for scripts, with built in masking for sensitive options. Provides a Splunk friendly formatted logger instance." license = "MIT" authors = ["dtomlinson "] -readme = "/Users/dtomlinson/git-repos/projects/panaetius/README.md" +readme = "/Users/dtomlinson/git-repos/projects/panaetius/README.rst" homepage = "https://github.com/dtomlinson91/panaetius" repository = "https://github.com/dtomlinson91/panaetius" -documentation = "https://github.com/dtomlinson91/panaetius" +documentation = "https://panaetius.readthedocs.io/en/latest/introduction.html" classifiers = [ "Environment :: Plugins", "Intended Audience :: Developers", diff --git a/setup.py b/setup.py index 8c99987..921f12b 100644 --- a/setup.py +++ b/setup.py @@ -15,9 +15,9 @@ install_requires = \ setup_kwargs = { 'name': 'panaetius', - 'version': '1.0', + 'version': '1.0.2', 'description': 'Python module to gracefully handle a .config file/environment variables for scripts, with built in masking for sensitive options. Provides a Splunk friendly logger instance.', - 'long_description': '# Author\n\nDaniel Tomlinson (dtomlinson@panaetius.co.uk)\n\n# Requires\n\n`>= python3.7`\n\n# Python requirements\n\n- toml = "^0.10.0"\n- pylite = "^0.1.0"\n\n# Documentation\n\n_soon_\n\n# Installation\n\n_soon_\n\n# Easy Way\n\n## Python\n\n### From pip\n\n### From local wheel\n\n### From source\n\n# Example Usage\n\n', + 'long_description': 'Author\n=======\n\nDaniel Tomlinson (dtomlinson@panaetius.co.uk)\n\nRequires\n=========\n\n`>= python3.7`\n\nPython requirements\n====================\n\n- toml = "^0.10.0"\n- pylite = "^0.1.0"\n\nDocumentation\n==============\n\nRead the documentation on `read the docs`_.\n\n.. _read the docs: https://panaetius.readthedocs.io/en/latest/introduction.html\n\nInstallation\n==============\n\nYou can install ..:obj:`panaetius`\n\nEasy Way\n=========\n\nPython\n-------\n\nFrom pip\n~~~~~~~~~\n\nFrom local wheel\n~~~~~~~~~~~~~~~~~\n\nFrom source\n~~~~~~~~~~~~\n\nExample Usage\n==============\n\n', 'author': 'dtomlinson', 'author_email': 'dtomlinson@panaetius.co.uk', 'url': 'https://github.com/dtomlinson91/panaetius', diff --git a/src/panaetius/__version__.py b/src/panaetius/__version__.py index 7e49527..cd7ca49 100644 --- a/src/panaetius/__version__.py +++ b/src/panaetius/__version__.py @@ -1 +1 @@ -__version__ = '1.0' +__version__ = '1.0.1' diff --git a/src/panaetius/db.py b/src/panaetius/db.py index 3eae95a..198b095 100644 --- a/src/panaetius/db.py +++ b/src/panaetius/db.py @@ -11,8 +11,41 @@ import panaetius class Mask: + + """Class to handle masking sensitive values in a config file + + Attributes + ---------- + config_contents : dict + A dict containing the contents of the config file. + config_path : str + The path to the config file. + config_var : str + The key corresponding to the config entry. + database : Pylite + A Pylite instance for the datbase. + entry : str + The result from the config file. Could either be a hash or the raw + value. + header : str + The __header__ which denotes where the config file is stored. + name : str + The key of the entry in the config file. + result : str + The value of the entry in the config file. + table_name : str + The sqlite table name. Defaults to the __header__ value. + """ + @property def hash(self): + """Property to determine the hash of a config entry. + + Returns + ------- + bytes + The hash as a bytes object. + """ try: if not self._hash_exists: pass @@ -30,15 +63,46 @@ class Mask: @property def salt(self): + """Property to detemine a random salt to use in creation of the hash. + + Returns + ------- + bytes + The salt as a bytes object. + """ self._salt = urandom(32) return self._salt @staticmethod def as_string(obj: bytes) -> str: + """Static method to return a string from a bytes object. + + Parameters + ---------- + obj : bytes + Bytes object to be converted to a string. + + Returns + ------- + str + The bytes object as a string. + """ return bytes.hex(obj) @staticmethod def fromhex(obj: str) -> bytes: + """Static method to create a bytes object from a string. + + Parameters + ---------- + obj : str + String object to be converted to bytes. + + Returns + ------- + bytes + The string object as bytes. + """ return bytes.fromhex(obj) @staticmethod @@ -53,6 +117,9 @@ class Mask: def __init__( self, config_path: str, config_contents: dict, config_var: str ): + """Summary + See :class:`~Mask` for parameters. + """ self.table: str = __header__ self.config_path = config_path self.config_contents = config_contents @@ -114,34 +181,34 @@ class Mask: self.entry[self.name], ) - def update_entries_in_db(self): + def _update_entries_in_db(self): self.database.remove(self.table, f'Name="{self.config_var}"') self._insert_entries() - def run_query(self, query: str): + def _run_query(self, query: str): cur = self.database.db.cursor() cur.execute(query) self.database.db.commit() self.result = cur.fetchall() return self - def get_all_items(self, where_clause: str = None): + def _get_all_items(self, where_clause: str = None): if where_clause is not None: self.result = self.database.get_items(self.table, where_clause) else: self.result = self.database.get_items(self.table) return self - def process(self): + def _process(self): if not self._check_entries(): # panaetius.logger.debug('does not exist') self._insert_entries() self._update_entries_in_config() - self.get_all_items() + self._get_all_items() # panaetius.logger.debug(f'returning: {self.result[0][3]}') return self.entry[self.name] else: - self.get_all_items(f'Name="{self.config_var}"') + self._get_all_items(f'Name="{self.config_var}"') if self.result[0][1] == self.entry[self.name]: # panaetius.logger.debug('exists and hash matches') # panaetius.logger.debug(f'returning: {self.result[0][3]}') @@ -151,9 +218,9 @@ class Mask: # panaetius.logger.debug( # f'file_hash={self.entry[self.name]}, {self.result[0][1]}' # ) - self.update_entries_in_db() + self._update_entries_in_db() self._update_entries_in_config() - self.get_all_items(f'Name="{self.config_var}"') + self._get_all_items(f'Name="{self.config_var}"') # panaetius.logger.debug(f'returning: {self.result[0][3]}') return self.entry[self.name] @@ -173,9 +240,17 @@ class Mask: c.close() def get_value(self): + """Get the true value from the database if it exists, create if it' + ' doesn't exist or update if the hash has changed. + + Returns + ------- + str + The result from the database. + """ # print(f'key in db {self.config_var}') self._get_database_file() self._open_database() self._get_table() - self.process() + self._process() return self.result[0][3] diff --git a/src/panaetius/library.py b/src/panaetius/library.py index 54538f1..9aa5b1e 100644 --- a/src/panaetius/library.py +++ b/src/panaetius/library.py @@ -56,24 +56,22 @@ def set_config( to define a list, but want to make sure that a string representation of a list will be loaded properly if it set as an environment variable. - Example: + Example: - `config.toml` has the following attribute set: - ``` - [package.users] - auth = ['user1', 'user2'] - ``` + *config.toml* has the following attribute set:: - If set as an environment variable you can pass this list as a string - and set check=list: + [package.users] + auth = ['user1', 'user2'] - Environment variable: - PACKAGE_USERS_AUTH = "['user1', 'user2']" + If set as an environment variable you can pass this list as a string + and set :code:`check=list`:: - Usage in code: - ```python - set_config(CONFIG, 'users.auth', check=list) - ``` + Environment variable: + PACKAGE_USERS_AUTH = "['user1', 'user2']" + + Usage in code:: + + set_config(CONFIG, 'users.auth', check=list) """ config_var = key.lower().replace('.', '_') if check is None: diff --git a/tests/test.py b/tests/test.py index 632d743..6b602a1 100644 --- a/tests/test.py +++ b/tests/test.py @@ -29,3 +29,5 @@ print(panaetius.CONFIG.config_file) # for i in panaetius.CONFIG.deferred_messages: # panaetius.logger.debug(i) + +panaetius.logger.info('some logging message')