From 0c03f327469465009d10399a877e48766cf3ee45 Mon Sep 17 00:00:00 2001 From: Daniel Tomlinson Date: Sun, 24 Oct 2021 07:06:40 +0100 Subject: [PATCH 01/10] adding latest --- tembo/exceptions.py | 4 +++ tembo/journal/pages.py | 79 +++++++++++++++++++++++++++++++----------- 2 files changed, 62 insertions(+), 21 deletions(-) diff --git a/tembo/exceptions.py b/tembo/exceptions.py index c8c6ad5..a71ea99 100644 --- a/tembo/exceptions.py +++ b/tembo/exceptions.py @@ -1 +1,5 @@ """Tembo exceptions.""" + + +class MismatchedTokenError(Exception): + pass diff --git a/tembo/journal/pages.py b/tembo/journal/pages.py index ba4a6e9..ebf8bbd 100644 --- a/tembo/journal/pages.py +++ b/tembo/journal/pages.py @@ -9,6 +9,7 @@ import jinja2 import pendulum from tembo import logger, CONFIG +from tembo.exceptions import MismatchedTokenError class PageCreator: @@ -83,23 +84,66 @@ class ScopedPageCreator(PageCreator): self.page_path = page_path self.filename = filename self.extension = extension + path_error: MismatchedTokenError | None = None + template_error: MismatchedTokenError | None = None + # get the path of the scoped page path = self._convert_to_path( self.base_path, self.page_path, self.filename, self.extension ) # substitute tokens in the filepath - path = pathlib.Path(self._substitute_tokens(str(path), user_input, name)) + try: + path = pathlib.Path(self._substitute_tokens(str(path), user_input, name)) + except MismatchedTokenError as mismatched_path_error: + path_error = mismatched_path_error # get the template file if template_filename is not None: # load the template file contents and substitute tokens template_contents = self._load_template(self.base_path, template_filename) - template_contents = self._substitute_tokens( - template_contents, user_input, name - ) + try: + template_contents = self._substitute_tokens( + template_contents, user_input, name + ) + except MismatchedTokenError as mismatched_template_error: + template_error = mismatched_template_error else: template_contents = "" + if path_error is not None or template_error is not None: + # self.__mismatched_token_error(path_error, template_error) + if path_error.args[0] > template_error.args[0]: + print("path_token_count > template_token_count") + elif template_error.args[0] > path_error.args[0]: + print("template_token_count > path_token_count") return ScopedPage(path, template_contents) + @staticmethod + def __mismatched_token_error( + path_error: MismatchedTokenError | None = None, + template_error: MismatchedTokenError | None = None, + ) -> None: + if isinstance(path_error, MismatchedTokenError): + path_token_count = path_error.args[0] + # logger.critical( + # "Your config specifies %s input tokens, you gave %s " "- exiting", + # path_error.args[0], + # path_error.args[1], + # ) + # raise SystemExit(1) + if isinstance(template_error, MismatchedTokenError): + template_token_count = template_error.args[0] + # logger.critical( + # "Your template specifies %s input tokens, you gave %s " "- exiting", + # template_error.args[0], + # template_error.args[1], + # ) + # raise SystemExit(1) + if path_token_count > template_token_count: + print("path_token_count > template_token_count") + elif template_token_count > path_token_count: + print("template_token_count > path_token_count") + raise SystemExit(1) + + # TODO: change the annotation to include the error def _substitute_tokens( self, tokenified_string: str, @@ -107,11 +151,14 @@ class ScopedPageCreator(PageCreator): name: str, ) -> str: # for a tokened string, substitute input, name and date tokens - tokenified_string = self.__substitute_input_tokens( - tokenified_string, user_input - ) - tokenified_string = self.__substitute_name_tokens(tokenified_string, name) - tokenified_string = self.__substitute_date_tokens(tokenified_string) + try: + tokenified_string = self.__substitute_input_tokens( + tokenified_string, user_input + ) + tokenified_string = self.__substitute_name_tokens(tokenified_string, name) + tokenified_string = self.__substitute_date_tokens(tokenified_string) + except MismatchedTokenError as mismatched_token_error: + raise mismatched_token_error return tokenified_string @staticmethod @@ -131,12 +178,7 @@ class ScopedPageCreator(PageCreator): if len(user_input) == 0: # if there's no user input, but the regex matches, raise error if len(input_extraction) > 0: - logger.critical( - "Your config/template specifies %s input tokens, you gave 0 " - "- exiting", - len(input_extraction), - ) - raise SystemExit(1) + raise MismatchedTokenError(len(input_extraction), 0) # if there aren't any tokens in the string, return the string return tokenified_string # if there is user input, check the number of tokens match the number passed in @@ -147,12 +189,7 @@ class ScopedPageCreator(PageCreator): ): # if there are input matches and they don't equal the number of input # tokens, raise error - logger.critical( - "Your config/template specifies %s input tokens, you gave %s - exiting", - len(input_extraction), - len(user_input), - ) - raise SystemExit(1) + raise MismatchedTokenError(len(input_extraction), len(user_input)) # if the length of both the input matches and the number of tokens match then # substitute each token with the user's input if len(user_input) > 0: From 5fb70300fb523cdac0ee3d860726437b38f9ea85 Mon Sep 17 00:00:00 2001 From: Daniel Tomlinson Date: Sun, 24 Oct 2021 18:28:36 +0100 Subject: [PATCH 02/10] adding latest --- tembo/journal/pages.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tembo/journal/pages.py b/tembo/journal/pages.py index ebf8bbd..72238a6 100644 --- a/tembo/journal/pages.py +++ b/tembo/journal/pages.py @@ -69,6 +69,9 @@ class ScopedPageCreator(PageCreator): self.page_path = "" self.filename = "" self.extension = "" + self.path_error: MismatchedTokenError | None = None + self.template_error: MismatchedTokenError | None = None + def create_page( self, @@ -84,8 +87,6 @@ class ScopedPageCreator(PageCreator): self.page_path = page_path self.filename = filename self.extension = extension - path_error: MismatchedTokenError | None = None - template_error: MismatchedTokenError | None = None # get the path of the scoped page path = self._convert_to_path( @@ -95,7 +96,7 @@ class ScopedPageCreator(PageCreator): try: path = pathlib.Path(self._substitute_tokens(str(path), user_input, name)) except MismatchedTokenError as mismatched_path_error: - path_error = mismatched_path_error + self.path_error = mismatched_path_error # get the template file if template_filename is not None: # load the template file contents and substitute tokens @@ -105,14 +106,14 @@ class ScopedPageCreator(PageCreator): template_contents, user_input, name ) except MismatchedTokenError as mismatched_template_error: - template_error = mismatched_template_error + self.template_error = mismatched_template_error else: template_contents = "" - if path_error is not None or template_error is not None: + if self.path_error is not None or self.template_error is not None: # self.__mismatched_token_error(path_error, template_error) - if path_error.args[0] > template_error.args[0]: + if self.path_error.args[0] > self.template_error.args[0]: print("path_token_count > template_token_count") - elif template_error.args[0] > path_error.args[0]: + elif self.template_error.args[0] > self.path_error.args[0]: print("template_token_count > path_token_count") return ScopedPage(path, template_contents) From 6b2b0a3b5eeadf3337f9fc8dc3485820bbaf49a4 Mon Sep 17 00:00:00 2001 From: Daniel Tomlinson Date: Sun, 24 Oct 2021 18:28:18 +0100 Subject: [PATCH 03/10] Auto stash before merge of "bugfix/token_mismatch" and "origin/bugfix/token_mismatch" --- .pythonversion => .python-version | 0 tembo/journal/pages.py | 69 +++++++++--------- tests/__pycache__/__init__.cpython-38.pyc | Bin 151 -> 148 bytes .../test_tembo.cpython-38-pytest-6.2.5.pyc | Bin 153 -> 150 bytes 4 files changed, 36 insertions(+), 33 deletions(-) rename .pythonversion => .python-version (100%) diff --git a/.pythonversion b/.python-version similarity index 100% rename from .pythonversion rename to .python-version diff --git a/tembo/journal/pages.py b/tembo/journal/pages.py index 72238a6..ae5b796 100644 --- a/tembo/journal/pages.py +++ b/tembo/journal/pages.py @@ -3,7 +3,7 @@ from __future__ import annotations from abc import ABCMeta, abstractmethod import pathlib import re -from typing import Tuple +from typing import Tuple, Literal import jinja2 import pendulum @@ -69,8 +69,8 @@ class ScopedPageCreator(PageCreator): self.page_path = "" self.filename = "" self.extension = "" - self.path_error: MismatchedTokenError | None = None - self.template_error: MismatchedTokenError | None = None + self.path_date_tokens: Tuple[int, int] = (0, 0) + self.template_date_tokens: Tuple[int, int] = (0, 0) def create_page( @@ -94,7 +94,9 @@ class ScopedPageCreator(PageCreator): ) # substitute tokens in the filepath try: - path = pathlib.Path(self._substitute_tokens(str(path), user_input, name)) + path = pathlib.Path( + self._substitute_tokens(str(path), user_input, name, "path") + ) except MismatchedTokenError as mismatched_path_error: self.path_error = mismatched_path_error # get the template file @@ -103,7 +105,7 @@ class ScopedPageCreator(PageCreator): template_contents = self._load_template(self.base_path, template_filename) try: template_contents = self._substitute_tokens( - template_contents, user_input, name + template_contents, user_input, name, "template" ) except MismatchedTokenError as mismatched_template_error: self.template_error = mismatched_template_error @@ -117,32 +119,32 @@ class ScopedPageCreator(PageCreator): print("template_token_count > path_token_count") return ScopedPage(path, template_contents) - @staticmethod - def __mismatched_token_error( - path_error: MismatchedTokenError | None = None, - template_error: MismatchedTokenError | None = None, - ) -> None: - if isinstance(path_error, MismatchedTokenError): - path_token_count = path_error.args[0] - # logger.critical( - # "Your config specifies %s input tokens, you gave %s " "- exiting", - # path_error.args[0], - # path_error.args[1], - # ) - # raise SystemExit(1) - if isinstance(template_error, MismatchedTokenError): - template_token_count = template_error.args[0] - # logger.critical( - # "Your template specifies %s input tokens, you gave %s " "- exiting", - # template_error.args[0], - # template_error.args[1], - # ) - # raise SystemExit(1) - if path_token_count > template_token_count: - print("path_token_count > template_token_count") - elif template_token_count > path_token_count: - print("template_token_count > path_token_count") - raise SystemExit(1) + # @staticmethod + # def __mismatched_token_error( + # path_error: MismatchedTokenError | None = None, + # template_error: MismatchedTokenError | None = None, + # ) -> None: + # if isinstance(path_error, MismatchedTokenError): + # path_token_count = path_error.args[0] + # # logger.critical( + # # "Your config specifies %s input tokens, you gave %s " "- exiting", + # # path_error.args[0], + # # path_error.args[1], + # # ) + # # raise SystemExit(1) + # if isinstance(template_error, MismatchedTokenError): + # template_token_count = template_error.args[0] + # # logger.critical( + # # "Your template specifies %s input tokens, you gave %s " "- exiting", + # # template_error.args[0], + # # template_error.args[1], + # # ) + # # raise SystemExit(1) + # if path_token_count > template_token_count: + # print("path_token_count > template_token_count") + # elif template_token_count > path_token_count: + # print("template_token_count > path_token_count") + # raise SystemExit(1) # TODO: change the annotation to include the error def _substitute_tokens( @@ -150,6 +152,7 @@ class ScopedPageCreator(PageCreator): tokenified_string: str, user_input: Tuple[str, ...] | Tuple[()], name: str, + token_type: Literal["path", "template"], ) -> str: # for a tokened string, substitute input, name and date tokens try: @@ -170,9 +173,9 @@ class ScopedPageCreator(PageCreator): tokenified_string = tokenified_string.replace(extracted_input, name) return tokenified_string - @staticmethod + # @staticmethod def __substitute_input_tokens( - tokenified_string: str, user_input: Tuple[str, ...] | Tuple[()] + self, tokenified_string: str, user_input: Tuple[str, ...] | Tuple[()] ) -> str: # find {inputN} tokens in string input_extraction = re.findall(r"(\{input\d*\})", tokenified_string) diff --git a/tests/__pycache__/__init__.cpython-38.pyc b/tests/__pycache__/__init__.cpython-38.pyc index 317d9d6aaf71c8b397ae2f32e1fe62869844b8e8..b550fe0f02174128a0407f6693b2cad3bd046cc2 100644 GIT binary patch delta 38 scmbQvIE9frl$V!_0SF%Tl}_ZgVYAfF$j?ok=qJuvS(1^TH!(*W0JlX7+W-In delta 41 vcmbQjIGvF@l$V!_0SNx<=TGFeVYkr_Elw>ep6DyiUX-7enp{#mF;g1=(-#ZI diff --git a/tests/__pycache__/test_tembo.cpython-38-pytest-6.2.5.pyc b/tests/__pycache__/test_tembo.cpython-38-pytest-6.2.5.pyc index 6bfba605c4f36b2dac1d7e1d38aa0f7acf251d64..91d170683e0a5178f624bc17758ff960e5cf6636 100644 GIT binary patch delta 38 scmbQqIE|4zl$V!_0SJWpODA&MuvzP8TEXl~vo0y{u0HenW6aWAK delta 41 vcmbQnIFpe(l$V!_0SLOY^CxoKu-oZ}7N-^!PxKXMFUrqKO)e>(n5hc@&4&w? From b1fda1fa8bdc7772186bf03de401b13497f3cf51 Mon Sep 17 00:00:00 2001 From: Daniel Tomlinson Date: Sun, 24 Oct 2021 21:56:04 +0100 Subject: [PATCH 04/10] adding latest --- TODO.todo | 4 + tembo/cli.py | 3 +- tembo/journal/pages.py | 164 ++++++++++++++++++++++------------------- 3 files changed, 96 insertions(+), 75 deletions(-) diff --git a/TODO.todo b/TODO.todo index 1287bf5..c956e6e 100644 --- a/TODO.todo +++ b/TODO.todo @@ -2,6 +2,10 @@ Functionality: ☐ Handle case where there are no scopes in the config and command is invoked. ☐ Have an `--example` flag to `new` that prints an example given in the `config.yml` ☐ Should be a `tembo new --list` to list all possible names. + ☐ When template not found, raise a Tembo error + currently raises jinja2.exceptions.TemplateNotFound on line 62, in _load_template + ☐ Add update notification? + check pypi for latest version and compare to current ✔ `TEMBO_CONFIG` should follow same pattern as other env vars and be a python string when read in @done(21-10-24 05:31) VSCode: diff --git a/tembo/cli.py b/tembo/cli.py index b26acec..1978be1 100644 --- a/tembo/cli.py +++ b/tembo/cli.py @@ -62,5 +62,6 @@ run.add_command(new) if __name__ == "__main__": # BUG: fix this bug where input tokens are mismatched - new(["meeting", "robs presentation", "meeting on gcp"]) + # new(["meeting", "robs presentation", "meeting on gcp"]) + new(["meeting", "a", "b", "c"]) # new(["meeting", "robs presentation"]) diff --git a/tembo/journal/pages.py b/tembo/journal/pages.py index ae5b796..4398af0 100644 --- a/tembo/journal/pages.py +++ b/tembo/journal/pages.py @@ -69,9 +69,9 @@ class ScopedPageCreator(PageCreator): self.page_path = "" self.filename = "" self.extension = "" - self.path_date_tokens: Tuple[int, int] = (0, 0) - self.template_date_tokens: Tuple[int, int] = (0, 0) - + # TODO: rename these to input tokens + more sensible + self.path_input_tokens: Tuple[int, int] = (0, 0) + self.template_input_tokens: Tuple[int, int] = (0, 0) def create_page( self, @@ -92,77 +92,84 @@ class ScopedPageCreator(PageCreator): path = self._convert_to_path( self.base_path, self.page_path, self.filename, self.extension ) + # substitute tokens in the filepath - try: - path = pathlib.Path( - self._substitute_tokens(str(path), user_input, name, "path") - ) - except MismatchedTokenError as mismatched_path_error: - self.path_error = mismatched_path_error + path = pathlib.Path( + self._substitute_tokens(str(path), user_input, name, "path") + ) + if sum(self.path_input_tokens) > 0: + _highest_input_token_in_path = max(self.path_input_tokens) + else: + _highest_input_token_in_path = 0 + # get the template file if template_filename is not None: # load the template file contents and substitute tokens template_contents = self._load_template(self.base_path, template_filename) - try: - template_contents = self._substitute_tokens( - template_contents, user_input, name, "template" - ) - except MismatchedTokenError as mismatched_template_error: - self.template_error = mismatched_template_error + template_contents = self._substitute_tokens( + template_contents, user_input, name, "template" + ) + if sum(self.template_input_tokens) > 0: + _highest_input_token_in_template = max(self.template_input_tokens) + else: + _highest_input_token_in_template = 0 else: template_contents = "" - if self.path_error is not None or self.template_error is not None: - # self.__mismatched_token_error(path_error, template_error) - if self.path_error.args[0] > self.template_error.args[0]: - print("path_token_count > template_token_count") - elif self.template_error.args[0] > self.path_error.args[0]: - print("template_token_count > path_token_count") + + self.__check_input_token_mismatch( + _highest_input_token_in_path, _highest_input_token_in_template + ) return ScopedPage(path, template_contents) - # @staticmethod - # def __mismatched_token_error( - # path_error: MismatchedTokenError | None = None, - # template_error: MismatchedTokenError | None = None, - # ) -> None: - # if isinstance(path_error, MismatchedTokenError): - # path_token_count = path_error.args[0] - # # logger.critical( - # # "Your config specifies %s input tokens, you gave %s " "- exiting", - # # path_error.args[0], - # # path_error.args[1], - # # ) - # # raise SystemExit(1) - # if isinstance(template_error, MismatchedTokenError): - # template_token_count = template_error.args[0] - # # logger.critical( - # # "Your template specifies %s input tokens, you gave %s " "- exiting", - # # template_error.args[0], - # # template_error.args[1], - # # ) - # # raise SystemExit(1) - # if path_token_count > template_token_count: - # print("path_token_count > template_token_count") - # elif template_token_count > path_token_count: - # print("template_token_count > path_token_count") - # raise SystemExit(1) + def __check_input_token_mismatch( + self, _highest_input_token_in_path: int, _highest_input_token_in_template: int + ) -> None: + _highest_input_token_count = max( + _highest_input_token_in_path, _highest_input_token_in_template + ) + if _highest_input_token_in_path < _highest_input_token_count: + logger.critical( + "Your config/template specifies %s input tokens, you gave %s " + "- exiting", + _highest_input_token_count, + self.path_input_tokens[0], + ) + raise SystemExit(1) + if _highest_input_token_in_path > _highest_input_token_count: + logger.warning( + "Your config/template specifies %s input tokens, you gave %s", + _highest_input_token_count, + self.path_input_tokens[0], + ) + if _highest_input_token_in_template < _highest_input_token_count: + logger.critical( + "Your config/template specifies %s input tokens, you gave %s " + "- exiting", + _highest_input_token_count, + self.template_input_tokens[0], + ) + raise SystemExit(1) + if _highest_input_token_in_template > _highest_input_token_count: + logger.warning( + "Your config/template specifies %s input tokens, you gave %s", + _highest_input_token_count, + self.template_input_tokens[0], + ) - # TODO: change the annotation to include the error def _substitute_tokens( self, tokenified_string: str, user_input: Tuple[str, ...] | Tuple[()], name: str, - token_type: Literal["path", "template"], + input_token_type: Literal["path", "template"], ) -> str: - # for a tokened string, substitute input, name and date tokens - try: - tokenified_string = self.__substitute_input_tokens( - tokenified_string, user_input - ) - tokenified_string = self.__substitute_name_tokens(tokenified_string, name) - tokenified_string = self.__substitute_date_tokens(tokenified_string) - except MismatchedTokenError as mismatched_token_error: - raise mismatched_token_error + """For a tokened string, substitute input, name and date tokens.""" + + tokenified_string = self.__substitute_input_tokens( + tokenified_string, user_input, input_token_type + ) + tokenified_string = self.__substitute_name_tokens(tokenified_string, name) + tokenified_string = self.__substitute_date_tokens(tokenified_string) return tokenified_string @staticmethod @@ -175,28 +182,35 @@ class ScopedPageCreator(PageCreator): # @staticmethod def __substitute_input_tokens( - self, tokenified_string: str, user_input: Tuple[str, ...] | Tuple[()] + self, + tokenified_string: str, + user_input: Tuple[str, ...] | Tuple[()], + input_token_type: Literal["path", "template"], ) -> str: - # find {inputN} tokens in string + """Find `{inputN}` tokens in string.""" + input_extraction = re.findall(r"(\{input\d*\})", tokenified_string) + # if there is no user input if len(user_input) == 0: - # if there's no user input, but the regex matches, raise error if len(input_extraction) > 0: - raise MismatchedTokenError(len(input_extraction), 0) + # if the regex matches, save the number of input tokens found + if input_token_type == "path": # noqa: bandit 105 + # TODO: change this to a dict instead of tuple + self.path_input_tokens = (len(input_extraction), 0) + if input_token_type == "template": # noqa: bandit 105 + self.template_input_tokens = (len(input_extraction), 0) # if there aren't any tokens in the string, return the string return tokenified_string - # if there is user input, check the number of tokens match the number passed in - if ( - len(input_extraction) > 0 - and len(input_extraction) != len(user_input) - and len(user_input) > 0 - ): - # if there are input matches and they don't equal the number of input - # tokens, raise error - raise MismatchedTokenError(len(input_extraction), len(user_input)) - # if the length of both the input matches and the number of tokens match then - # substitute each token with the user's input + + # if there is user input if len(user_input) > 0: + # save the number of input tokens, and the number of user inputs + if input_token_type == "path": # noqa: bandit 105 + self.path_input_tokens = (len(input_extraction), len(user_input)) + elif input_token_type == "template": # noqa: bandit 105 + self.template_input_tokens = (len(input_extraction), len(user_input)) + + # sbustitute the input token for the user's input for extracted_input, input_value in zip(input_extraction, user_input): tokenified_string = tokenified_string.replace( extracted_input, input_value @@ -205,7 +219,9 @@ class ScopedPageCreator(PageCreator): @staticmethod def __substitute_date_tokens(tokenified_string: str) -> str: - # find any {d:%d-%M-%Y} tokens + """Find any {d:%d-%M-%Y} tokens.""" + + # extract the full token string date_extraction_token = re.findall(r"(\{d\:[^}]*\})", tokenified_string) for extracted_token in date_extraction_token: # extract the inner %d-%M-%Y only From 2d91c9d62d2284f19265c957e62d20d7bca3d9ef Mon Sep 17 00:00:00 2001 From: Daniel Tomlinson Date: Sun, 24 Oct 2021 22:34:58 +0100 Subject: [PATCH 05/10] adding latest --- TODO.todo | 40 +++++++++++++++---------- tembo/journal/pages.py | 68 ++++++++++++++++-------------------------- 2 files changed, 50 insertions(+), 58 deletions(-) diff --git a/TODO.todo b/TODO.todo index c956e6e..cbe844e 100644 --- a/TODO.todo +++ b/TODO.todo @@ -8,30 +8,40 @@ Functionality: check pypi for latest version and compare to current ✔ `TEMBO_CONFIG` should follow same pattern as other env vars and be a python string when read in @done(21-10-24 05:31) +Bug: + ☐ tokens + Say we have input0 and input3 in file path + and we have input1 and input2 in template + it only recognises this as 2 inputs total, not four. + passing in tembo new meeting a b will work and input0=input1, input3=input2 + VSCode: ☐ Look at Logging: - ☐ How to raise + debug an exception? - ☐ Document how to raise a logger.critical instead of exception - in a try, except you can just do logger.critical(exec_info=1) to print the stack -Documentation: - ☐ Document usage of Panaetius in a module - Using the logger, initialising with the config path etc - ✘ Uses Pendulum tokens: https://pendulum.eustace.io/docs/#tokens @cancelled(21-10-24 05:32) - ☐ Uses `strftime` tokens: - ☐ Document latest typing. + Documentation: + ☐ Document usage of Panaetius in a module + Using the logger, initialising with the config path etc + ✘ Uses Pendulum tokens: https://pendulum.eustace.io/docs/#tokens @cancelled(21-10-24 05:32) + ☐ Uses `strftime` tokens: + ☐ Document latest typing. ☐ Using from `__future__` with `|` ☐ `using Tuple[str, ...]` ☐ `Sequence` vs `Collection` - ☐ Document how to do docstrings in python. Don't document `__init__` do it in class. - Should update the default gist to hide the `__init__` messages - ☐ Document using jinja2 briefly and link to Tembo (link to ) - Tembo: - ☐ Document creating new Tembo config + ☐ Document how to do docstrings in python. Don't document `__init__` do it in class. + Should update the default gist to hide the `__init__` messages + ☐ Document using jinja2 briefly and link to Tembo (link to ) + + Logging: + ☐ How to raise + debug an exception? + ☐ Document how to raise a logger.critical instead of exception + in a try, except you can just do logger.critical(exec_info=1) to print the stack + + Tembo: + ☐ Document creating new Tembo config ☐ ~/tembo needs creating ☐ ~/tembo/.config ☐ ~/tembo/.templates ☐ ~/tembo/logs - ☐ Document how to overwrite these with ENV vars + ☐ Document how to overwrite these with ENV vars diff --git a/tembo/journal/pages.py b/tembo/journal/pages.py index 4398af0..f27afd1 100644 --- a/tembo/journal/pages.py +++ b/tembo/journal/pages.py @@ -9,7 +9,6 @@ import jinja2 import pendulum from tembo import logger, CONFIG -from tembo.exceptions import MismatchedTokenError class PageCreator: @@ -70,8 +69,8 @@ class ScopedPageCreator(PageCreator): self.filename = "" self.extension = "" # TODO: rename these to input tokens + more sensible - self.path_input_tokens: Tuple[int, int] = (0, 0) - self.template_input_tokens: Tuple[int, int] = (0, 0) + self.path_input_token_counts = {"config": 0, "user": 0} + self.template_input_token_counts = {"config": 0, "user": 0} def create_page( self, @@ -97,10 +96,6 @@ class ScopedPageCreator(PageCreator): path = pathlib.Path( self._substitute_tokens(str(path), user_input, name, "path") ) - if sum(self.path_input_tokens) > 0: - _highest_input_token_in_path = max(self.path_input_tokens) - else: - _highest_input_token_in_path = 0 # get the template file if template_filename is not None: @@ -109,51 +104,34 @@ class ScopedPageCreator(PageCreator): template_contents = self._substitute_tokens( template_contents, user_input, name, "template" ) - if sum(self.template_input_tokens) > 0: - _highest_input_token_in_template = max(self.template_input_tokens) - else: - _highest_input_token_in_template = 0 else: template_contents = "" - self.__check_input_token_mismatch( - _highest_input_token_in_path, _highest_input_token_in_template - ) + self.__check_input_token_mismatch() return ScopedPage(path, template_contents) - def __check_input_token_mismatch( - self, _highest_input_token_in_path: int, _highest_input_token_in_template: int - ) -> None: - _highest_input_token_count = max( - _highest_input_token_in_path, _highest_input_token_in_template + def __check_input_token_mismatch(self) -> None: + _max_config_input_token_count = max( + self.path_input_token_counts["config"], + self.template_input_token_counts["config"], ) - if _highest_input_token_in_path < _highest_input_token_count: + _max_user_input_token_count = max( + self.path_input_token_counts["user"], + self.template_input_token_counts["user"], + ) + if _max_user_input_token_count < _max_config_input_token_count: logger.critical( "Your config/template specifies %s input tokens, you gave %s " "- exiting", - _highest_input_token_count, - self.path_input_tokens[0], + _max_config_input_token_count, + _max_user_input_token_count, ) raise SystemExit(1) - if _highest_input_token_in_path > _highest_input_token_count: + if _max_user_input_token_count > _max_config_input_token_count: logger.warning( "Your config/template specifies %s input tokens, you gave %s", - _highest_input_token_count, - self.path_input_tokens[0], - ) - if _highest_input_token_in_template < _highest_input_token_count: - logger.critical( - "Your config/template specifies %s input tokens, you gave %s " - "- exiting", - _highest_input_token_count, - self.template_input_tokens[0], - ) - raise SystemExit(1) - if _highest_input_token_in_template > _highest_input_token_count: - logger.warning( - "Your config/template specifies %s input tokens, you gave %s", - _highest_input_token_count, - self.template_input_tokens[0], + _max_config_input_token_count, + _max_user_input_token_count, ) def _substitute_tokens( @@ -196,9 +174,11 @@ class ScopedPageCreator(PageCreator): # if the regex matches, save the number of input tokens found if input_token_type == "path": # noqa: bandit 105 # TODO: change this to a dict instead of tuple - self.path_input_tokens = (len(input_extraction), 0) + self.path_input_token_counts["config"] = len(input_extraction) + self.path_input_token_counts["user"] = 0 if input_token_type == "template": # noqa: bandit 105 - self.template_input_tokens = (len(input_extraction), 0) + self.template_input_token_counts["config"] = len(input_extraction) + self.template_input_token_counts["user"] = 0 # if there aren't any tokens in the string, return the string return tokenified_string @@ -206,9 +186,11 @@ class ScopedPageCreator(PageCreator): if len(user_input) > 0: # save the number of input tokens, and the number of user inputs if input_token_type == "path": # noqa: bandit 105 - self.path_input_tokens = (len(input_extraction), len(user_input)) + self.path_input_token_counts["config"] = len(input_extraction) + self.path_input_token_counts["user"] = len(user_input) elif input_token_type == "template": # noqa: bandit 105 - self.template_input_tokens = (len(input_extraction), len(user_input)) + self.template_input_token_counts["config"] = len(input_extraction) + self.template_input_token_counts["user"] = len(user_input) # sbustitute the input token for the user's input for extracted_input, input_value in zip(input_extraction, user_input): From a19866e3d8e465fab6c5e8e2c812f8951bb3f19e Mon Sep 17 00:00:00 2001 From: Daniel Tomlinson Date: Mon, 25 Oct 2021 02:48:16 +0100 Subject: [PATCH 06/10] updating .gitgnore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 2dc658c..34243a4 100644 --- a/.gitignore +++ b/.gitignore @@ -137,4 +137,6 @@ dmypy.json # Cython debug symbols cython_debug/ +# custom .vscode/ +**/__pycache__ From 87c6e99eed63208b3f81e3bcc96cb6ba2d7c4ade Mon Sep 17 00:00:00 2001 From: Daniel Tomlinson Date: Mon, 25 Oct 2021 02:51:12 +0100 Subject: [PATCH 07/10] removing __pycache__ --- tests/__pycache__/__init__.cpython-38.pyc | Bin 148 -> 0 bytes .../test_tembo.cpython-38-pytest-6.2.5.pyc | Bin 150 -> 0 bytes 2 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 tests/__pycache__/__init__.cpython-38.pyc delete mode 100644 tests/__pycache__/test_tembo.cpython-38-pytest-6.2.5.pyc diff --git a/tests/__pycache__/__init__.cpython-38.pyc b/tests/__pycache__/__init__.cpython-38.pyc deleted file mode 100644 index b550fe0f02174128a0407f6693b2cad3bd046cc2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 148 zcmWIL<>g`kf=7L&i6Hthh(HF6K#l_t7qb9~6oz01O-8?!3`HPe1o6vKKO;XkRX?R9 zKQ|{cuQ)$XKRvTVwg`k0-^rWL=gQLL?8o3AjbiSi&=m~3PUi1CZpd Date: Mon, 25 Oct 2021 02:51:22 +0100 Subject: [PATCH 08/10] updating todo --- TODO.todo | 1 + 1 file changed, 1 insertion(+) diff --git a/TODO.todo b/TODO.todo index cbe844e..cfa36ec 100644 --- a/TODO.todo +++ b/TODO.todo @@ -3,6 +3,7 @@ Functionality: ☐ Have an `--example` flag to `new` that prints an example given in the `config.yml` ☐ Should be a `tembo new --list` to list all possible names. ☐ When template not found, raise a Tembo error + ☐ Convert spaces to underscores in filepath currently raises jinja2.exceptions.TemplateNotFound on line 62, in _load_template ☐ Add update notification? check pypi for latest version and compare to current From d5879f711fffa5dc23a4717b2329fa6c3951d2c9 Mon Sep 17 00:00:00 2001 From: Daniel Tomlinson Date: Mon, 25 Oct 2021 03:46:11 +0100 Subject: [PATCH 09/10] adding latest --- tembo/cli.py | 3 +-- tembo/journal/pages.py | 52 ++++++++++++++++++++++++++++++++++++++---- 2 files changed, 49 insertions(+), 6 deletions(-) diff --git a/tembo/cli.py b/tembo/cli.py index 1978be1..e5810e6 100644 --- a/tembo/cli.py +++ b/tembo/cli.py @@ -46,7 +46,6 @@ def new(scope, inputs, dry_run): template_filename=str(user_scope["template_filename"]) ) scoped_page.save_to_disk(dry_run=dry_run) - tembo.logger.info("Saved %s to disk", scoped_page.path) raise SystemExit(0) if not _name_found and len(tembo.CONFIG.scopes) > 0: tembo.logger.warning("Command %s not found in config.yml - exiting", scope) @@ -63,5 +62,5 @@ run.add_command(new) if __name__ == "__main__": # BUG: fix this bug where input tokens are mismatched # new(["meeting", "robs presentation", "meeting on gcp"]) - new(["meeting", "a", "b", "c"]) + new(["meeting", "a", "b", "c", "d"]) # new(["meeting", "robs presentation"]) diff --git a/tembo/journal/pages.py b/tembo/journal/pages.py index f27afd1..98a8379 100644 --- a/tembo/journal/pages.py +++ b/tembo/journal/pages.py @@ -68,6 +68,7 @@ class ScopedPageCreator(PageCreator): self.page_path = "" self.filename = "" self.extension = "" + self._all_input_tokens: list[str] = [] # TODO: rename these to input tokens + more sensible self.path_input_token_counts = {"config": 0, "user": 0} self.template_input_token_counts = {"config": 0, "user": 0} @@ -87,6 +88,11 @@ class ScopedPageCreator(PageCreator): self.filename = filename self.extension = extension + # verify the user input matches the number of input tokens in the + # config/templates + self._all_input_tokens = self.__get_input_tokens(template_filename) + self.__verify_input_tokens(user_input) + # get the path of the scoped page path = self._convert_to_path( self.base_path, self.page_path, self.filename, self.extension @@ -110,6 +116,40 @@ class ScopedPageCreator(PageCreator): self.__check_input_token_mismatch() return ScopedPage(path, template_contents) + def __get_input_tokens(self, template_filename: str | None) -> list[str]: + path = str( + pathlib.Path( + self.base_path, self.page_path, self.filename, self.extension + ).expanduser() + ) + if template_filename is not None: + template_contents = self._load_template(self.base_path, template_filename) + else: + template_contents = "" + # get the input tokens from both the path and the template + all_input_tokens = [] + for tokenified_string in (path, template_contents): + all_input_tokens.extend(re.findall(r"(\{input\d*\})", tokenified_string)) + return sorted(all_input_tokens) + + def __verify_input_tokens(self, user_input: Tuple[str, ...] | Tuple[()]) -> None: + if len(self._all_input_tokens) != len(user_input): + logger.critical( + "Your config/template specifies %s input tokens, you gave %s", + len(self._all_input_tokens), + len(user_input), + ) + raise SystemExit(1) + + def __substitute_input_tokens__( + self, + tokenified_string: str, + user_input: Tuple[str, ...] | Tuple[()], + ) -> str: + for input_value, extracted_token in zip(user_input, self._all_input_tokens): + tokenified_string = tokenified_string.replace(extracted_token, input_value) + return tokenified_string + def __check_input_token_mismatch(self) -> None: _max_config_input_token_count = max( self.path_input_token_counts["config"], @@ -142,9 +182,12 @@ class ScopedPageCreator(PageCreator): input_token_type: Literal["path", "template"], ) -> str: """For a tokened string, substitute input, name and date tokens.""" - - tokenified_string = self.__substitute_input_tokens( - tokenified_string, user_input, input_token_type + # TODO: fn to get tokens from file and template + # tokenified_string = self.__substitute_input_tokens( + # tokenified_string, user_input, input_token_type + # ) + tokenified_string = self.__substitute_input_tokens__( + tokenified_string, user_input ) tokenified_string = self.__substitute_name_tokens(tokenified_string, name) tokenified_string = self.__substitute_date_tokens(tokenified_string) @@ -247,9 +290,10 @@ class ScopedPage(Page): if not scoped_note_file.exists(): with scoped_note_file.open("w", encoding="utf-8") as scoped_page: scoped_page.write(self.page_content) + logger.info("Saved %s to disk", self.path) else: logger.info("%s already exists - skipping.", self.path) - raise SystemExit(0) + raise SystemExit(0) if __name__ == "__main__": From 794e6bf8fa89ed453d68bb9274ff0edfb4e2814a Mon Sep 17 00:00:00 2001 From: Daniel Tomlinson Date: Mon, 25 Oct 2021 04:24:28 +0100 Subject: [PATCH 10/10] adding latest --- TODO.todo | 2 + tembo/journal/pages.py | 124 +++++++++-------------------------------- 2 files changed, 29 insertions(+), 97 deletions(-) diff --git a/TODO.todo b/TODO.todo index cfa36ec..0f5c56b 100644 --- a/TODO.todo +++ b/TODO.todo @@ -46,3 +46,5 @@ Logging: ☐ ~/tembo/.templates ☐ ~/tembo/logs ☐ Document how to overwrite these with ENV vars + ☐ have a git repo with all the above already configured and walk user through + clone the repo, delete .git, git init, configure and add git origin diff --git a/tembo/journal/pages.py b/tembo/journal/pages.py index 98a8379..8609377 100644 --- a/tembo/journal/pages.py +++ b/tembo/journal/pages.py @@ -3,7 +3,7 @@ from __future__ import annotations from abc import ABCMeta, abstractmethod import pathlib import re -from typing import Tuple, Literal +from typing import Tuple import jinja2 import pendulum @@ -69,9 +69,6 @@ class ScopedPageCreator(PageCreator): self.filename = "" self.extension = "" self._all_input_tokens: list[str] = [] - # TODO: rename these to input tokens + more sensible - self.path_input_token_counts = {"config": 0, "user": 0} - self.template_input_token_counts = {"config": 0, "user": 0} def create_page( self, @@ -88,10 +85,10 @@ class ScopedPageCreator(PageCreator): self.filename = filename self.extension = extension - # verify the user input matches the number of input tokens in the + # verify the user input length matches the number of input tokens in the # config/templates - self._all_input_tokens = self.__get_input_tokens(template_filename) - self.__verify_input_tokens(user_input) + self._all_input_tokens = self._get_input_tokens(template_filename) + self._verify_input_tokens(user_input) # get the path of the scoped page path = self._convert_to_path( @@ -100,7 +97,7 @@ class ScopedPageCreator(PageCreator): # substitute tokens in the filepath path = pathlib.Path( - self._substitute_tokens(str(path), user_input, name, "path") + self._substitute_tokens(str(path), user_input, name) ) # get the template file @@ -108,15 +105,14 @@ class ScopedPageCreator(PageCreator): # load the template file contents and substitute tokens template_contents = self._load_template(self.base_path, template_filename) template_contents = self._substitute_tokens( - template_contents, user_input, name, "template" + template_contents, user_input, name ) else: template_contents = "" - self.__check_input_token_mismatch() return ScopedPage(path, template_contents) - def __get_input_tokens(self, template_filename: str | None) -> list[str]: + def _get_input_tokens(self, template_filename: str | None) -> list[str]: path = str( pathlib.Path( self.base_path, self.page_path, self.filename, self.extension @@ -132,7 +128,7 @@ class ScopedPageCreator(PageCreator): all_input_tokens.extend(re.findall(r"(\{input\d*\})", tokenified_string)) return sorted(all_input_tokens) - def __verify_input_tokens(self, user_input: Tuple[str, ...] | Tuple[()]) -> None: + def _verify_input_tokens(self, user_input: Tuple[str, ...] | Tuple[()]) -> None: if len(self._all_input_tokens) != len(user_input): logger.critical( "Your config/template specifies %s input tokens, you gave %s", @@ -141,7 +137,25 @@ class ScopedPageCreator(PageCreator): ) raise SystemExit(1) - def __substitute_input_tokens__( + def _substitute_tokens( + self, + tokenified_string: str, + user_input: Tuple[str, ...] | Tuple[()], + name: str, + ) -> str: + """For a tokened string, substitute input, name and date tokens.""" + # TODO: fn to get tokens from file and template + # tokenified_string = self.__substitute_input_tokens( + # tokenified_string, user_input, input_token_type + # ) + tokenified_string = self.__substitute_input_tokens( + tokenified_string, user_input + ) + tokenified_string = self.__substitute_name_tokens(tokenified_string, name) + tokenified_string = self.__substitute_date_tokens(tokenified_string) + return tokenified_string + + def __substitute_input_tokens( self, tokenified_string: str, user_input: Tuple[str, ...] | Tuple[()], @@ -150,49 +164,6 @@ class ScopedPageCreator(PageCreator): tokenified_string = tokenified_string.replace(extracted_token, input_value) return tokenified_string - def __check_input_token_mismatch(self) -> None: - _max_config_input_token_count = max( - self.path_input_token_counts["config"], - self.template_input_token_counts["config"], - ) - _max_user_input_token_count = max( - self.path_input_token_counts["user"], - self.template_input_token_counts["user"], - ) - if _max_user_input_token_count < _max_config_input_token_count: - logger.critical( - "Your config/template specifies %s input tokens, you gave %s " - "- exiting", - _max_config_input_token_count, - _max_user_input_token_count, - ) - raise SystemExit(1) - if _max_user_input_token_count > _max_config_input_token_count: - logger.warning( - "Your config/template specifies %s input tokens, you gave %s", - _max_config_input_token_count, - _max_user_input_token_count, - ) - - def _substitute_tokens( - self, - tokenified_string: str, - user_input: Tuple[str, ...] | Tuple[()], - name: str, - input_token_type: Literal["path", "template"], - ) -> str: - """For a tokened string, substitute input, name and date tokens.""" - # TODO: fn to get tokens from file and template - # tokenified_string = self.__substitute_input_tokens( - # tokenified_string, user_input, input_token_type - # ) - tokenified_string = self.__substitute_input_tokens__( - tokenified_string, user_input - ) - tokenified_string = self.__substitute_name_tokens(tokenified_string, name) - tokenified_string = self.__substitute_date_tokens(tokenified_string) - return tokenified_string - @staticmethod def __substitute_name_tokens(tokenified_string: str, name: str) -> str: # find any {name} tokens and substitute for the name value @@ -201,47 +172,6 @@ class ScopedPageCreator(PageCreator): tokenified_string = tokenified_string.replace(extracted_input, name) return tokenified_string - # @staticmethod - def __substitute_input_tokens( - self, - tokenified_string: str, - user_input: Tuple[str, ...] | Tuple[()], - input_token_type: Literal["path", "template"], - ) -> str: - """Find `{inputN}` tokens in string.""" - - input_extraction = re.findall(r"(\{input\d*\})", tokenified_string) - # if there is no user input - if len(user_input) == 0: - if len(input_extraction) > 0: - # if the regex matches, save the number of input tokens found - if input_token_type == "path": # noqa: bandit 105 - # TODO: change this to a dict instead of tuple - self.path_input_token_counts["config"] = len(input_extraction) - self.path_input_token_counts["user"] = 0 - if input_token_type == "template": # noqa: bandit 105 - self.template_input_token_counts["config"] = len(input_extraction) - self.template_input_token_counts["user"] = 0 - # if there aren't any tokens in the string, return the string - return tokenified_string - - # if there is user input - if len(user_input) > 0: - # save the number of input tokens, and the number of user inputs - if input_token_type == "path": # noqa: bandit 105 - self.path_input_token_counts["config"] = len(input_extraction) - self.path_input_token_counts["user"] = len(user_input) - elif input_token_type == "template": # noqa: bandit 105 - self.template_input_token_counts["config"] = len(input_extraction) - self.template_input_token_counts["user"] = len(user_input) - - # sbustitute the input token for the user's input - for extracted_input, input_value in zip(input_extraction, user_input): - tokenified_string = tokenified_string.replace( - extracted_input, input_value - ) - return tokenified_string - @staticmethod def __substitute_date_tokens(tokenified_string: str) -> str: """Find any {d:%d-%M-%Y} tokens."""