From 75adf89839cad4222afa4b13c564e988b90057d4 Mon Sep 17 00:00:00 2001 From: obar1 <387386+obar1@users.noreply.github.com> Date: Sun, 8 Sep 2024 15:35:59 +0200 Subject: [PATCH] Feature/closes 87 (#120) close #87 --- .gitignore | 2 +- demo.sh | 3 +++ zero_to_one_hundred/factories/sb_factory.py | 17 +++++++++++++-- zero_to_one_hundred/factories/ztoh_factory.py | 20 +++++++++++++++--- .../factories/ztoh_factory_provider.py | 4 +++- zero_to_one_hundred/models/map.py | 4 ++-- zero_to_one_hundred/models/meta_book.py | 19 +++++++++++------ zero_to_one_hundred/models/metadata.py | 4 ++-- zero_to_one_hundred/models/readme_md.py | 2 +- zero_to_one_hundred/models/section.py | 4 ++-- zero_to_one_hundred/models/toc.py | 21 +++++++++++++++---- .../processors/refresh_links_processor.py | 2 +- .../processors/refresh_map_processor.py | 2 +- zero_to_one_hundred/runner.py | 14 +++++++++---- .../tests/test_ztoh/test_map.py | 14 ++++++------- .../tests/test_ztoh/test_readme_md.py | 4 ++-- .../tests/test_ztoh/test_section.py | 4 ++-- .../tests/test_ztoh/test_validator.py | 13 ++++++++---- .../tests/tests_sb/resources/map.yaml | 6 ++++++ .../tests/tests_sb/test_metadata.py | 10 ++++----- .../tests/tests_sb/test_toc.py | 13 ++++++++---- zero_to_one_hundred/validator/validator.py | 11 ++++++---- .../views/markdown_renderer.py | 2 +- 23 files changed, 136 insertions(+), 59 deletions(-) diff --git a/.gitignore b/.gitignore index fe951bb..e8d6f9a 100644 --- a/.gitignore +++ b/.gitignore @@ -19,7 +19,7 @@ venv/ 978*/ map.md repo/ -toc.md +toc*.md *.yaml safaribooks/ diff --git a/demo.sh b/demo.sh index d05d090..6c78841 100644 --- a/demo.sh +++ b/demo.sh @@ -47,6 +47,7 @@ done <<< "$content" ls -1R 0to100 + cp toc.md toc_0to100.md } function 0to100_sb { @@ -66,6 +67,8 @@ function 0to100_sb { ./main_sb.py refresh_toc ls -1R 978* + cp toc.md toc_0to100_sb.md + } setup diff --git a/zero_to_one_hundred/factories/sb_factory.py b/zero_to_one_hundred/factories/sb_factory.py index db7c9c7..2488f79 100644 --- a/zero_to_one_hundred/factories/sb_factory.py +++ b/zero_to_one_hundred/factories/sb_factory.py @@ -1,3 +1,4 @@ +import argparse from enum import Enum from zero_to_one_hundred.configs.sb_config_map import SBConfigMap @@ -27,9 +28,21 @@ def __init__( self.process_fs = process_fs def get_processor(self, args): - cmd = args[1] + parser = argparse.ArgumentParser(description="Run 0to100_sb.") + valid_cmds = list(p.name for p in self.SUPPORTED_PROCESSOR) + parser.add_argument( + "cmd", + type=str, + help=f'command, must be {" ".join(valid_cmds)}', + choices=valid_cmds, + ) + parser.add_argument("p1", type=str, help="arg p1", nargs="?", default=None) + + args = parser.parse_args(args[1:]) + cmd = args.cmd + p1 = args.p1 if cmd == SBFactory.SUPPORTED_PROCESSOR.snatch_book.name: - http_url = args[2] + http_url = p1 yield self.snatch_book_processor(http_url) yield self.refresh_toc_processor() elif cmd == SBFactory.SUPPORTED_PROCESSOR.refresh_toc.name: diff --git a/zero_to_one_hundred/factories/ztoh_factory.py b/zero_to_one_hundred/factories/ztoh_factory.py index 500c90a..d419978 100644 --- a/zero_to_one_hundred/factories/ztoh_factory.py +++ b/zero_to_one_hundred/factories/ztoh_factory.py @@ -1,3 +1,4 @@ +import argparse from enum import Enum from zero_to_one_hundred.configs.ztoh_config_map import ZTOHConfigMap @@ -34,12 +35,25 @@ def __init__( self.process_fs = process_fs def get_processor(self, args): - cmd = args[1] + parser = argparse.ArgumentParser(description="Run 0to100.") + valid_cmds = list(p.name for p in self.SUPPORTED_PROCESSOR) + parser.add_argument( + "cmd", + type=str, + help=f'command, must be {" ".join(valid_cmds)}', + choices=valid_cmds, + ) + parser.add_argument("p1", type=str, help="arg p1", nargs="?", default=None) + + args = parser.parse_args(args[1:]) + cmd = args.cmd + p1 = args.p1 + if cmd == ZTOHFactory.SUPPORTED_PROCESSOR.create_section.name: - yield self.create_section_processor(args[2]) + yield self.create_section_processor(p1) yield self.refresh_map_processor() elif cmd == ZTOHFactory.SUPPORTED_PROCESSOR.done_section.name: - yield self.done_section_processor(args[2]) + yield self.done_section_processor(p1) yield self.refresh_map_processor() elif cmd == ZTOHFactory.SUPPORTED_PROCESSOR.refresh_map.name: yield self.refresh_map_processor() diff --git a/zero_to_one_hundred/factories/ztoh_factory_provider.py b/zero_to_one_hundred/factories/ztoh_factory_provider.py index f46e3e1..1e2979b 100644 --- a/zero_to_one_hundred/factories/ztoh_factory_provider.py +++ b/zero_to_one_hundred/factories/ztoh_factory_provider.py @@ -18,4 +18,6 @@ def provide(self) -> ZTOHFactory: config_map_type = config_map.get_type if config_map_type == ZTOH_MAP: return ZTOHFactory(config_map, self.persist_fs, self.process_fs) - raise NotImplementedError(f"NotImplementedError {config_map_type}") + raise NotImplementedError( + f"NotImplementedError {config_map_type}, check the files contents of {config_map.map_yaml_path}" + ) diff --git a/zero_to_one_hundred/models/map.py b/zero_to_one_hundred/models/map.py index 53e5f48..252d72d 100644 --- a/zero_to_one_hundred/models/map.py +++ b/zero_to_one_hundred/models/map.py @@ -35,7 +35,7 @@ def get_sections(self): res = sorted(self.sections, key=lambda s: s.get_readme_md_time()) return res - def asMarkDown(self) -> str: + def as_mark_down(self) -> str: lf_char = "\n" def get_legend_as_md(self): @@ -50,7 +50,7 @@ def get_legend_as_md(self): {get_legend_as_md(self)} -{lf_char.join((section.asMarkDown() for section in self.get_sections()))} +{lf_char.join((section.as_mark_down() for section in self.get_sections()))} """ return txt.replace(" ", "") diff --git a/zero_to_one_hundred/models/meta_book.py b/zero_to_one_hundred/models/meta_book.py index 12a0cff..8574ac0 100644 --- a/zero_to_one_hundred/models/meta_book.py +++ b/zero_to_one_hundred/models/meta_book.py @@ -88,27 +88,27 @@ def write(self): self.persist_fs.make_dirs(self.config_map.get_download_engine_books_path) self.persist_fs.make_dirs(self.contents_path) except Exception as e: - Validator.print_DDD(e) + Validator.print_e(e) try: self.write_img() except Exception as e: - Validator.print_DDD(e) + Validator.print_e(e) try: self.write_epub() except Exception as e: - Validator.print_DDD(e) + Validator.print_e(e) try: self.write_metadata() except Exception as e: - Validator.print_DDD(e) + Validator.print_e(e) try: self.write_pdf(self.path_epub) except Exception as e: - Validator.print_DDD(e) + Validator.print_e(e) try: self.write_splitter_pdf(self.path_pdf, self.config_map.get_split_pdf_pages) except Exception as e: - Validator.print_DDD(e) + Validator.print_e(e) @classmethod def get_isbn(cls, http_url): @@ -135,3 +135,10 @@ def path_as_md(self, a_path): use relative path and convert " " to %20 """ return a_path.replace(" ", "%20") + + @property + def get_matching_icon_as_md(self): + icons = self.config_map.get_legend_icons + + res = [i.icon for i in icons if re.search(i.regex, self.http_url)] + return " ".join(res) diff --git a/zero_to_one_hundred/models/metadata.py b/zero_to_one_hundred/models/metadata.py index 9d5181c..0687c8c 100644 --- a/zero_to_one_hundred/models/metadata.py +++ b/zero_to_one_hundred/models/metadata.py @@ -30,7 +30,7 @@ def __init__( self.metadata: dict = self.read() def __repr__(self): - return f"MetaBook {self.isbn} {self.http_url} {self.asMarkDown()}" + return f"MetaBook {self.isbn} {self.http_url} {self.as_mark_down()}" @staticmethod def get_page_perc(metadata_dict: dict): @@ -79,7 +79,7 @@ def get_metadata(self): sorted_dict = dict(sorted(metadata_dict.items())) return sorted_dict - def asMarkDown(self) -> str: + def as_mark_down(self) -> str: # handle nasty URL in MD m: dict = self.get_metadata() url = m.get("url") diff --git a/zero_to_one_hundred/models/readme_md.py b/zero_to_one_hundred/models/readme_md.py index e272827..c4b452f 100644 --- a/zero_to_one_hundred/models/readme_md.py +++ b/zero_to_one_hundred/models/readme_md.py @@ -26,7 +26,7 @@ def __init__( def __repr__(self): return f"ReadMeMD {self.readme_md} {self.http_url} {self.dir_name}" - def asMarkDown(self): + def as_mark_down(self): return f"ReadMeMD {self.readme_md}, {self.dir_name} {self.http_url}" def write(self, txt=None): diff --git a/zero_to_one_hundred/models/section.py b/zero_to_one_hundred/models/section.py index 2280de1..2363065 100644 --- a/zero_to_one_hundred/models/section.py +++ b/zero_to_one_hundred/models/section.py @@ -38,7 +38,7 @@ def __init__( def __repr__(self): return f"Section {self.http_url} {self.dir_readme_md} {self.is_done} {self.dir_name}" - def asMarkDown(self): + def as_mark_down(self): return ( "1. " + self.get_id_name @@ -177,7 +177,7 @@ def get_header(line): if len(not_null) > 1: # take first one header found res = not_null[1] except Exception as e: - Validator.print_DDD(e) + Validator.print_e(e) res = "FIXME: " return res diff --git a/zero_to_one_hundred/models/toc.py b/zero_to_one_hundred/models/toc.py index db4f7bf..64c61c6 100644 --- a/zero_to_one_hundred/models/toc.py +++ b/zero_to_one_hundred/models/toc.py @@ -43,7 +43,7 @@ def build_from_dirs( logging.info(res) return res - def asMarkDown(self): + def as_mark_down(self): def flatten_meta_book(meta_book: MetaBook): logging.info(f"flatten_meta_book {meta_book}") txt = "|".join( @@ -53,11 +53,22 @@ def flatten_meta_book(meta_book: MetaBook): f"[`xyz`]({meta_book.contents_path_as_md})", f"{meta_book.metadata.as_mark_down()}", f"{meta_book.metadata.status}", + f"{meta_book.get_matching_icon_as_md}", ] ) return "|" + txt + "|" + lf_char = "\n" + + def get_legend_as_md(self): + txt: str = """ +## legend: +""" + txt += lf_char + txt += self.config_map.get_legend_icons_as_md + return txt + flattened_meta_book = [flatten_meta_book(mb) for mb in self.meta_books] backslash_n_char = "\n" @@ -67,13 +78,15 @@ def flatten_meta_book(meta_book: MetaBook): # TOC ## `{len(self.meta_books)}` metabook ### {self.process_fs.get_now()} -| ISBN | img | `meta-contents` | `json-contents` | `status` | -|--- |--- |--- |--- |--- | +{get_legend_as_md(self)} + +| ISBN | img | `meta-contents` | `json-contents` | `status` | `icons` +|--- |--- |--- |--- |--- |--- | {backslash_n_char.join(flattened_meta_book)} """ ) return md def write(self): - md = self.asMarkDown() + md = self.as_mark_down() return self.persist_fs.write_file(self.readme_md, md) diff --git a/zero_to_one_hundred/processors/refresh_links_processor.py b/zero_to_one_hundred/processors/refresh_links_processor.py index 38cc8a3..978ae79 100644 --- a/zero_to_one_hundred/processors/refresh_links_processor.py +++ b/zero_to_one_hundred/processors/refresh_links_processor.py @@ -37,4 +37,4 @@ def process(self): try: s.refresh_links() except Exception as e: - Validator.print_DDD(e) + Validator.print_e(e) diff --git a/zero_to_one_hundred/processors/refresh_map_processor.py b/zero_to_one_hundred/processors/refresh_map_processor.py index 6cf2ccd..26f3a5a 100644 --- a/zero_to_one_hundred/processors/refresh_map_processor.py +++ b/zero_to_one_hundred/processors/refresh_map_processor.py @@ -32,4 +32,4 @@ def process(self): self.persist_fs.list_dirs(self.config_map.get_repo_path), ), ) - map.write(map.asMarkDown()) + map.write(map.as_mark_down()) diff --git a/zero_to_one_hundred/runner.py b/zero_to_one_hundred/runner.py index ed60c17..01c1ae9 100644 --- a/zero_to_one_hundred/runner.py +++ b/zero_to_one_hundred/runner.py @@ -1,5 +1,6 @@ # pylint: disable=W0106,R1710 from typing import List +from typing import Union, TypeVar from zero_to_one_hundred.exceptions.errors import SomeError from zero_to_one_hundred.factories.a_factory import AFactory @@ -15,16 +16,21 @@ def run_core(argv: List[str], factory_provider: AFactoryProvider): factory_provider (AFactoryProvider): a factory_type """ - factory: AFactory + + T = TypeVar("T", bound=AFactory) + factory: Union[AFactory, T] try: factory = factory_provider.provide() [processor.process() for processor in factory.get_processor(argv) if processor] except SomeError as e: - Validator.print_DDD(e) + Validator.print_e(e) return except FileNotFoundError as e: - Validator.print_DDD(e) + Validator.print_e(e) + return + except NotImplementedError as e: + Validator.print_e(e) return except Exception as e: - Validator.print_DDD(e) + Validator.print_e(e) factory.help_processor().process() diff --git a/zero_to_one_hundred/tests/test_ztoh/test_map.py b/zero_to_one_hundred/tests/test_ztoh/test_map.py index 72057e1..c71151a 100644 --- a/zero_to_one_hundred/tests/test_ztoh/test_map.py +++ b/zero_to_one_hundred/tests/test_ztoh/test_map.py @@ -10,7 +10,7 @@ # pylint: disable=W0102 -def test_asMarkDown( +def test_as_mark_down( get_config_map: ZTOHConfigMap, persist_fs, process_fs, @@ -21,7 +21,7 @@ def test_asMarkDown( for http_url in http_urls ] actual = Map(get_config_map, persist_fs, sections=sections) - current = actual.asMarkDown() + current = actual.as_mark_down() expected = """ # map toc.md, 2 ## legend: @@ -32,7 +32,7 @@ def test_asMarkDown( assert str_relaxed(current) == str_relaxed(expected) -def test_asMarkDown_0( +def test_as_mark_down_0( get_config_map_sorted_0: ZTOHConfigMap, persist_fs, process_fs, @@ -47,7 +47,7 @@ def test_asMarkDown_0( for http_url in http_urls ] actual = Map(get_config_map_sorted_0, persist_fs, sections=sections) - current = actual.asMarkDown() + current = actual.as_mark_down() expected = """ # map toc.md, 3 ## legend: @@ -60,7 +60,7 @@ def test_asMarkDown_0( assert str_relaxed(current) == str_relaxed(expected) -def test_asMarkDown_1( +def test_as_mark_down_1( get_config_map_sorted_1: ZTOHConfigMap, persist_fs, process_fs, @@ -75,7 +75,7 @@ def test_asMarkDown_1( for http_url in http_urls ] actual = Map(get_config_map_sorted_1, persist_fs, sections=sections) - current = actual.asMarkDown() + current = actual.as_mark_down() expected = """ # map toc.md, 3 ## legend: @@ -100,7 +100,7 @@ def test_write( for http_url in http_urls ] actual = Map(get_config_map, persist_fs, sections=sections) - txt = actual.asMarkDown() + txt = actual.as_mark_down() with Patcher(allow_root_user=False) as patcher: res = actual.write(txt) assert res > 0 diff --git a/zero_to_one_hundred/tests/test_ztoh/test_readme_md.py b/zero_to_one_hundred/tests/test_ztoh/test_readme_md.py index 49364f9..2e1acae 100644 --- a/zero_to_one_hundred/tests/test_ztoh/test_readme_md.py +++ b/zero_to_one_hundred/tests/test_ztoh/test_readme_md.py @@ -12,7 +12,7 @@ def test_refresh_links(get_config_map, persist_fs, process_fs, http_url_1): ) -def test_asMarkDown(get_config_map, persist_fs, process_fs, http_url_1): +def test_as_mark_down(get_config_map, persist_fs, process_fs, http_url_1): actual = ReadMeMD( get_config_map, persist_fs, @@ -20,7 +20,7 @@ def test_asMarkDown(get_config_map, persist_fs, process_fs, http_url_1): Section.from_http_url_to_dir, http_url_1, ) - current = actual.asMarkDown() + current = actual.as_mark_down() assert ( current == "ReadMeMD ./0to100/https§§§cloud.google.com§abc/readme.md, https§§§cloud.google.com§abc https://cloud.google.com/abc" diff --git a/zero_to_one_hundred/tests/test_ztoh/test_section.py b/zero_to_one_hundred/tests/test_ztoh/test_section.py index 55b5041..782f6ec 100644 --- a/zero_to_one_hundred/tests/test_ztoh/test_section.py +++ b/zero_to_one_hundred/tests/test_ztoh/test_section.py @@ -93,9 +93,9 @@ def test_gcp_get_format_as_md(get_gcp_config_map, persist_fs, process_fs): assert actual.get_matching_icon_as_md == """:snake:""" -def test_asMarkDown(get_config_map, persist_fs, process_fs, http_url_1): +def test_as_mark_down(get_config_map, persist_fs, process_fs, http_url_1): actual = Section(get_config_map, persist_fs, process_fs, http_url_1) - current = actual.asMarkDown() + current = actual.as_mark_down() assert str_relaxed(current) == str_relaxed( "1. [`here`](./0to100/https§§§cloud.google.com§abc/readme.md) `wip`" ) diff --git a/zero_to_one_hundred/tests/test_ztoh/test_validator.py b/zero_to_one_hundred/tests/test_ztoh/test_validator.py index 71de37d..3bbb593 100644 --- a/zero_to_one_hundred/tests/test_ztoh/test_validator.py +++ b/zero_to_one_hundred/tests/test_ztoh/test_validator.py @@ -6,9 +6,14 @@ from zero_to_one_hundred.validator.validator import Validator -def test_is_valid_http__pass__fail(): - # pass - assert Validator.is_valid_http("https://code.google") is None - # fail +def test_is_valid_http_pass0(): + assert Validator.is_valid_http("https" + "://code.google") is None + + +def test_is_valid_http_pass1(): + assert Validator.is_valid_http("http" + "://www.cloudskillsboost.google/") is None + + +def test_is_valid_http_fail(): with pytest.raises(NotURLFormatError): Validator.is_valid_http("code.google") diff --git a/zero_to_one_hundred/tests/tests_sb/resources/map.yaml b/zero_to_one_hundred/tests/tests_sb/resources/map.yaml index cefd05c..1cdbea3 100644 --- a/zero_to_one_hundred/tests/tests_sb/resources/map.yaml +++ b/zero_to_one_hundred/tests/tests_sb/resources/map.yaml @@ -6,3 +6,9 @@ configs: oreilly_username: "username" oreilly_userpassword: "userpassword" split_pdf_pages: 100 +legend: + type: "sb" + icons: + - name: Book + icon: ":cyclone:" + regex: "978" diff --git a/zero_to_one_hundred/tests/tests_sb/test_metadata.py b/zero_to_one_hundred/tests/tests_sb/test_metadata.py index 19814a7..5cc534c 100644 --- a/zero_to_one_hundred/tests/tests_sb/test_metadata.py +++ b/zero_to_one_hundred/tests/tests_sb/test_metadata.py @@ -30,7 +30,7 @@ def test_get_page_perc(get_config_map, persist_fs, process_fs, http_oreilly_1): assert actual == "n/a" -def test_asMarkDown( +def test_as_mark_down( get_config_map, persist_fs, process_fs, http_oreilly_1, oreilly_isbn_1 ): actual = Metadata( @@ -41,7 +41,7 @@ def test_asMarkDown( http_oreilly_1, ) - assert str_relaxed(actual.asMarkDown()) == str_relaxed( + assert str_relaxed(actual.as_mark_down()) == str_relaxed( """ { "isbn":"9780135956977",
@@ -55,7 +55,7 @@ def test_asMarkDown( data = '{ "abc": "123", "def": "456"}' actual.metadata = json.loads(data) - assert str_relaxed(actual.asMarkDown()) == str_relaxed( + assert str_relaxed(actual.as_mark_down()) == str_relaxed( """ { "abc": "123",
@@ -76,8 +76,8 @@ def test_asMarkDown( } """ actual.metadata = json.loads(data) - print(actual.asMarkDown()) - assert str_relaxed(actual.asMarkDown()) == str_relaxed( + print(actual.as_mark_down()) + assert str_relaxed(actual.as_mark_down()) == str_relaxed( """ { "abc": "123",
diff --git a/zero_to_one_hundred/tests/tests_sb/test_toc.py b/zero_to_one_hundred/tests/tests_sb/test_toc.py index 858cded..37fbad2 100644 --- a/zero_to_one_hundred/tests/tests_sb/test_toc.py +++ b/zero_to_one_hundred/tests/tests_sb/test_toc.py @@ -30,7 +30,7 @@ def test_init(get_config_map, persist_fs, process_fs, http_oreilly_1): @pytest.mark.usefixtures("mock_time") -def test_asMarkDown( +def test_as_mark_down( get_config_map, persist_fs, process_fs, http_oreilly_1, http_oreilly_2 ): metabooks = [ @@ -53,12 +53,17 @@ def test_asMarkDown( process_fs, [], ) - current = actual.asMarkDown() + current = actual.as_mark_down() expected = """ # TOC ## `0` metabook ### 2099/01/01 - 00:00:00 -| ISBN | img | `meta-contents` | `json-contents` | `status` | -|--- |--- |--- |--- |--- | +## legend +legendicons +Book +cyclone +| ISBN | img | `meta-contents` | `json-contents` | `status` | `icons` | +|--- |--- |--- |--- |--- |--- | + """ assert str_relaxed("".join(current)) == str_relaxed("".join(expected)) diff --git a/zero_to_one_hundred/validator/validator.py b/zero_to_one_hundred/validator/validator.py index 0474813..735142e 100644 --- a/zero_to_one_hundred/validator/validator.py +++ b/zero_to_one_hundred/validator/validator.py @@ -1,5 +1,7 @@ import logging import traceback +import re + from zero_to_one_hundred.exceptions.errors import NotURLFormatError @@ -7,10 +9,11 @@ class Validator: @classmethod def is_valid_http(cls, url: str): - if not url.startswith("https://"): + pattern = r"^[^https?:\/\/].*" + if re.match(pattern, url) is not None: raise NotURLFormatError(f"{url} not valid") @classmethod - def print_DDD(cls, e: Exception): - logging.info(traceback.format_exc()) - logging.info(f"DDD issue with {e}") + def print_e(cls, e: Exception): + logging.exception(traceback.format_exc()) + logging.exception(f"#DDD issue with {e}") diff --git a/zero_to_one_hundred/views/markdown_renderer.py b/zero_to_one_hundred/views/markdown_renderer.py index 3b54caf..2e15bb8 100644 --- a/zero_to_one_hundred/views/markdown_renderer.py +++ b/zero_to_one_hundred/views/markdown_renderer.py @@ -7,7 +7,7 @@ class MarkdownRenderer(ABC): """ @abstractmethod - def asMarkDown(self) -> str: + def as_mark_down(self) -> str: pass @staticmethod