From e98f1ad80dad03f391724f8cf3453534b7eb920a Mon Sep 17 00:00:00 2001 From: Daniel Tomlinson Date: Sat, 20 Nov 2021 16:33:16 +0000 Subject: [PATCH 1/2] workaround #1 --- .vscode/settings.json | 17 +---------------- README.md | 9 +++++++-- panaetius/logging.py | 23 ++++++++++++++++++++--- 3 files changed, 28 insertions(+), 21 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 7c32b0d..9f07c66 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -4,20 +4,5 @@ "python.linting.enabled": true, "python.pythonPath": ".venv/bin/python", "restructuredtext.confPath": "${workspaceFolder}/docs/source", - "peacock.color": "#307E6A", - "workbench.colorCustomizations": { - "editorGroup.border": "#3ea389", - "panel.border": "#3ea389", - "sash.hoverBorder": "#3ea389", - "sideBar.border": "#3ea389", - "statusBar.background": "#307e6a", - "statusBar.foreground": "#e7e7e7", - "statusBarItem.hoverBackground": "#3ea389", - "statusBarItem.remoteBackground": "#307e6a", - "statusBarItem.remoteForeground": "#e7e7e7", - "titleBar.activeBackground": "#307e6a", - "titleBar.activeForeground": "#e7e7e7", - "titleBar.inactiveBackground": "#307e6a99", - "titleBar.inactiveForeground": "#e7e7e799" - } + "peacock.color": "#307E6A" } diff --git a/README.md b/README.md index f933352..76339ec 100644 --- a/README.md +++ b/README.md @@ -32,16 +32,21 @@ See Tembo for an example: str: raise NotImplementedError + @property + @abstractmethod + def logging_level(self) -> str: + raise NotImplementedError + @abstractmethod def __init__(self, logging_level: str): raise NotImplementedError @@ -119,8 +124,12 @@ class SimpleLogger(LoggingData): '"%(levelname)s",\n\t"message": "%(message)s"\n}', ) + @property + def logging_level(self) -> str: + return self._logging_level + def __init__(self, logging_level: str = "INFO"): - self.logging_level = logging_level + self._logging_level = logging_level class AdvancedLogger(LoggingData): @@ -133,8 +142,12 @@ class AdvancedLogger(LoggingData): '"%(levelname)s",\n\t"message": "%(message)s"\n}', ) + @property + def logging_level(self) -> str: + return self._logging_level + def __init__(self, logging_level: str = "INFO"): - self.logging_level = logging_level + self._logging_level = logging_level class CustomLogger(LoggingData): @@ -142,6 +155,10 @@ class CustomLogger(LoggingData): def format(self) -> str: return str(self._format) + @property + def logging_level(self) -> str: + return self._logging_level + def __init__(self, logging_format: str, logging_level: str = "INFO"): - self.logging_level = logging_level + self._logging_level = logging_level self._format = logging_format From 22935237bea15cd168fbafe0d0b3a59a9f9ae9a2 Mon Sep 17 00:00:00 2001 From: Daniel Tomlinson Date: Sat, 20 Nov 2021 18:34:35 +0000 Subject: [PATCH 2/2] updating latest --- README.md | 4 +- duties.py | 164 ++++++++++++++++++++++++++++++++++++++++++++++--- pyproject.toml | 13 ++++ setup.py | 4 +- 4 files changed, 172 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 76339ec..3c357a5 100644 --- a/README.md +++ b/README.md @@ -42,9 +42,9 @@ from panaetius.exceptions import LoggingDirectoryDoesNotExistException if (config_path := os.environ.get("TEMBO_CONFIG")) is not None: - CONFIG: Any = panaetius.Config("tembo", config_path, skip_header_init=True) # type: ignore + CONFIG: Any = panaetius.Config("tembo", config_path, skip_header_init=True) else: - CONFIG: Any = panaetius.Config( # type: ignore + CONFIG = panaetius.Config( "tembo", "~/tembo/.config", skip_header_init=True ) diff --git a/duties.py b/duties.py index 26e49a1..c0cb0be 100644 --- a/duties.py +++ b/duties.py @@ -1,13 +1,15 @@ from __future__ import annotations +import importlib import os import pathlib import re import shutil +import sys +from io import StringIO from duty import duty - PACKAGE_NAME = "panaetius" @@ -16,6 +18,11 @@ def update_deps(ctx, dry: bool = False): """ Update the dependencies using Poetry. + Args: + ctx: The context instance (passed automatically). + dry (bool, optional) = If True will update the `poetry.lock` without updating the + dependencies themselves. Defaults to False. + Example: `duty update_deps dry=False` """ @@ -28,21 +35,32 @@ def update_deps(ctx, dry: bool = False): @duty def test(ctx): - """Run tests using pytest""" - pytest_results = ctx.run(["pytest", "-v"]) + """ + Run tests using pytest. + + Args: + ctx: The context instance (passed automatically). + """ + pytest_results = ctx.run(["pytest", "-v"], pty=True) print(pytest_results) @duty def coverage(ctx): """ - Generate a coverage HTML report. + Generate a coverage report and save to XML and HTML. + + Args: + ctx: The context instance (passed automatically). Example: `duty coverage` """ ctx.run(["coverage", "run", "--source", PACKAGE_NAME, "-m", "pytest"]) + res = ctx.run(["coverage", "report"], pty=True) + print(res) ctx.run(["coverage", "html"]) + ctx.run(["coverage", "xml"]) @duty @@ -51,6 +69,7 @@ def version(ctx, bump: str = "patch"): Bump the version using Poetry and update _version.py. Args: + ctx: The context instance (passed automatically). bump (str, optional) = poetry version flag. Available options are: patch, minor, major, prepatch, preminor, premajor, prerelease. Defaults to patch. @@ -68,8 +87,7 @@ def version(ctx, bump: str = "patch"): version_file = pathlib.Path(PACKAGE_NAME) / "_version.py" with version_file.open("w", encoding="utf-8") as version_file: version_file.write( - f'"""Module containing the version of {PACKAGE_NAME}."""\n\n' - + f'__version__ = "{new_version.group(1)}"\n' + f'"""Module containing the version of {PACKAGE_NAME}."""\n\n' + f'__version__ = "{new_version.group(1)}"\n' ) print(f"Bumped _version.py to {new_version.group(1)}") @@ -77,7 +95,10 @@ def version(ctx, bump: str = "patch"): @duty def build(ctx): """ - Build with poetry and extract the `setup.py` and copy to project root. + Build with poetry and extract the setup.py and copy to project root. + + Args: + ctx: The context instance (passed automatically). Example: `duty build` @@ -108,6 +129,9 @@ def export(ctx): """ Export the dependencies to a requirements.txt file. + Args: + ctx: The context instance (passed automatically). + Example: `duty export` """ @@ -142,11 +166,12 @@ def export(ctx): @duty -def publish(ctx, password:str): +def publish(ctx, password: str): """ Publish the package to pypi.org. Args: + ctx: The context instance (passed automatically). password (str): pypi.org password. Example: @@ -160,6 +185,127 @@ def publish(ctx, password:str): print(publish_result) +@duty(silent=True) +def clean(ctx): + """ + Delete temporary files. + + Args: + ctx: The context instance (passed automatically). + """ + ctx.run("rm -rf .mypy_cache") + ctx.run("rm -rf .pytest_cache") + ctx.run("rm -rf tests/.pytest_cache") + ctx.run("rm -rf build") + ctx.run("rm -rf dist") + ctx.run("rm -rf pip-wheel-metadata") + ctx.run("rm -rf site") + ctx.run("rm -rf coverage.xml") + ctx.run("rm -rf pytest.xml") + ctx.run("rm -rf htmlcov") + ctx.run("find . -iname '.coverage*' -not -name .coveragerc | xargs rm -rf") + ctx.run("find . -type d -name __pycache__ | xargs rm -rf") + ctx.run("find . -name '*.rej' -delete") + + +@duty +def format(ctx): + """ + Format code using Black and isort. + + Args: + ctx: The context instance (passed automatically). + """ + res = ctx.run(["black", "--line-length=99", PACKAGE_NAME], pty=True, title="Running Black") + print(res) + + res = ctx.run(["isort", PACKAGE_NAME]) + print(res) + + +@duty(pre=["check_code_quality", "check_types", "check_docs", "check_dependencies"]) +def check(ctx): + """ + Check the code quality, check types, check documentation builds and check dependencies for vulnerabilities. + + Args: + ctx: The context instance (passed automatically). + """ + + +@duty +def check_code_quality(ctx): + """ + Check the code quality using prospector. + + Args: + ctx: The context instance (passed automatically). + """ + ctx.run(["prospector", PACKAGE_NAME], pty=True, title="Checking code quality with prospector") + + +@duty +def check_types(ctx): + """ + Check the types using mypy. + + Args: + ctx: The context instance (passed automatically). + """ + ctx.run(["mypy", PACKAGE_NAME], pty=True, title="Checking types with MyPy") + + +@duty +def check_docs(ctx): + """ + Check the documentation builds successfully. + + Args: + ctx: The context instance (passed automatically). + """ + ctx.run(["mkdocs", "build"], title="Building documentation") + + +@duty +def check_dependencies(ctx): + """ + Check dependencies with safety for vulnerabilities. + + Args: + ctx: The context instance (passed automatically). + """ + for module in sys.modules: + if module.startswith("safety.") or module == "safety": + del sys.modules[module] + + importlib.invalidate_caches() + + from safety import safety + from safety.formatter import report + from safety.util import read_requirements + + requirements = ctx.run( + "poetry export --dev --without-hashes", + title="Exporting dependencies as requirements", + allow_overrides=False, + ) + + def check_vulns(): + packages = list(read_requirements(StringIO(requirements))) + vulns = safety.check(packages=packages, ignore_ids="41002", key="", db_mirror="", cached=False, proxy={}) + output_report = report(vulns=vulns, full=True, checked_packages=len(packages)) + print(vulns) + if vulns: + print(output_report) + + ctx.run( + check_vulns, + stdin=requirements, + title="Checking dependencies", + pty=True, + ) + + def rm_tree(directory: pathlib.Path): """ Recursively delete a directory and all its contents. @@ -167,7 +313,7 @@ def rm_tree(directory: pathlib.Path): Args: directory (pathlib.Path): The directory to delete. """ - for child in directory.glob('*'): + for child in directory.glob("*"): if child.is_file(): child.unlink() else: diff --git a/pyproject.toml b/pyproject.toml index 974e63f..7875796 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,6 +37,19 @@ coverage = "^6.0.2" duty = "^0.7.0" types-PyYAML = "^6.0.1" +[tool.black] +line-length = 120 + +[tool.isort] +line-length = 120 +not_skip = "__init__.py" +multi_line_output = 3 +force_single_line = false +balanced_wrapping = true +default_section = "THIRDPARTY" +known_first_party = "duty" +include_trailing_comma = true + [build-system] requires = ["poetry>=0.12"] build-backend = "poetry.masonry.api" diff --git a/setup.py b/setup.py index acec652..3646deb 100644 --- a/setup.py +++ b/setup.py @@ -12,9 +12,9 @@ install_requires = \ setup_kwargs = { 'name': 'panaetius', - 'version': '2.3.1', + 'version': '2.3.2', 'description': 'Python module to gracefully handle a .config file/environment variables for scripts, with built in masking for sensitive options. Provides a Splunk friendly formatted logger instance.', - 'long_description': '# Panaetius\n\nThis package provides:\n\n- Functionality to read user variables from a `config.yml` or environment variables.\n- A convenient default logging formatter printing `json` that can save to disk and rotate.\n- Utility functions.\n\n## Config\n\n### options\n\n#### skip_header_init\n\nIf `skip_header_init=True` then the `config_path` will not use the `header_variable` as the\nsub-directory in the `config_path`.\n\nE.g\n\n`CONFIG = panaetius.Config("tembo", "~/tembo/.config", skip_header_init=True)`\n\nWill look in `~/tembo/config/config.yml`.\n\nIf `skip_header_init=False` then would look in `~/tembo/config/tembo/config.yml`.\n\n### Module\n\nConvenient to place in a package/sub-package `__init__.py`.\n\nSee Tembo for an example: \n\nExample snippet to use in a module:\n\n```python\nimport os\n\nimport panaetius\nfrom panaetius.exceptions import LoggingDirectoryDoesNotExistException\n\n\nif (config_path := os.environ.get("TEMBO_CONFIG")) is not None:\n CONFIG = panaetius.Config("tembo", config_path, skip_header_init=True)\nelse:\n CONFIG = panaetius.Config("tembo", "~/tembo/.config", skip_header_init=True)\n\n\npanaetius.set_config(CONFIG, "base_path", "~/tembo")\npanaetius.set_config(CONFIG, "template_path", "~/tembo/.templates")\npanaetius.set_config(CONFIG, "scopes", {})\npanaetius.set_config(CONFIG, "logging.level", "DEBUG")\npanaetius.set_config(CONFIG, "logging.path")\n\ntry:\n logger = panaetius.set_logger(\n CONFIG, panaetius.SimpleLogger(logging_level=CONFIG.logging_level)\n )\nexcept LoggingDirectoryDoesNotExistException:\n _LOGGING_PATH = CONFIG.logging_path\n CONFIG.logging_path = ""\n logger = panaetius.set_logger(\n CONFIG, panaetius.SimpleLogger(logging_level=CONFIG.logging_level)\n )\n logger.warning("Logging directory %s does not exist", _LOGGING_PATH)\n\n```\n\nThis means in `./tembo/cli/cli.py` you can\n\n```python\nimport tembo.cli\n\n# access the CONFIG instance + variables from the config.yml\ntembo.cli.CONFIG\n```\n\n\n## Utility Functions\n\n### Squasher\n\nSquashes a json object or Python dictionary into a single level dictionary.\n', + 'long_description': '# Panaetius\n\nThis package provides:\n\n- Functionality to read user variables from a `config.yml` or environment variables.\n- A convenient default logging formatter printing `json` that can save to disk and rotate.\n- Utility functions.\n\n## Config\n\n### options\n\n#### skip_header_init\n\nIf `skip_header_init=True` then the `config_path` will not use the `header_variable` as the\nsub-directory in the `config_path`.\n\nE.g\n\n`CONFIG = panaetius.Config("tembo", "~/tembo/.config", skip_header_init=True)`\n\nWill look in `~/tembo/config/config.yml`.\n\nIf `skip_header_init=False` then would look in `~/tembo/config/tembo/config.yml`.\n\n### Module\n\nConvenient to place in a package/sub-package `__init__.py`.\n\nSee Tembo for an example: \n\nExample snippet to use in a module:\n\n```python\n"""Subpackage that contains the CLI application."""\n\nimport os\nfrom typing import Any\n\nimport panaetius\nfrom panaetius.exceptions import LoggingDirectoryDoesNotExistException\n\n\nif (config_path := os.environ.get("TEMBO_CONFIG")) is not None:\n CONFIG: Any = panaetius.Config("tembo", config_path, skip_header_init=True)\nelse:\n CONFIG = panaetius.Config(\n "tembo", "~/tembo/.config", skip_header_init=True\n )\n\n\npanaetius.set_config(CONFIG, "base_path", "~/tembo")\npanaetius.set_config(CONFIG, "template_path", "~/tembo/.templates")\npanaetius.set_config(CONFIG, "scopes", {})\npanaetius.set_config(CONFIG, "logging.level", "DEBUG")\npanaetius.set_config(CONFIG, "logging.path")\n\ntry:\n logger = panaetius.set_logger(\n CONFIG, panaetius.SimpleLogger(logging_level=CONFIG.logging_level)\n )\nexcept LoggingDirectoryDoesNotExistException:\n _LOGGING_PATH = CONFIG.logging_path\n CONFIG.logging_path = ""\n logger = panaetius.set_logger(\n CONFIG, panaetius.SimpleLogger(logging_level=CONFIG.logging_level)\n )\n logger.warning("Logging directory %s does not exist", _LOGGING_PATH)\n\n```\n\nThis means in `./tembo/cli/cli.py` you can\n\n```python\nimport tembo.cli\n\n# access the CONFIG instance + variables from the config.yml\ntembo.cli.CONFIG\n```\n\n\n## Utility Functions\n\n### Squasher\n\nSquashes a json object or Python dictionary into a single level dictionary.\n', 'author': 'dtomlinson', 'author_email': 'dtomlinson@panaetius.co.uk', 'maintainer': None,