From 59dd4d2d98849d3b72c51d73780b265cb2127543 Mon Sep 17 00:00:00 2001 From: lordlabuckdas <55460753+lordlabuckdas@users.noreply.github.com> Date: Tue, 24 Aug 2021 23:27:28 +0530 Subject: [PATCH] add docstrings to class and fix ambiguous descriptions --- snare/cloner.py | 20 ++++++++++++++------ snare/html_handler.py | 4 +++- snare/middlewares.py | 4 +++- snare/server.py | 4 +++- snare/tanner_handler.py | 4 +++- snare/utils/asyncmock.py | 4 +++- snare/utils/logger.py | 5 +++-- snare/utils/snare_helpers.py | 10 +++++++--- 8 files changed, 39 insertions(+), 16 deletions(-) diff --git a/snare/cloner.py b/snare/cloner.py index 9092bf43..7eadb7c9 100644 --- a/snare/cloner.py +++ b/snare/cloner.py @@ -22,8 +22,10 @@ class BaseCloner: + """Abstract base class for all core functions of cloner""" + def __init__(self, root: str, max_depth: int, css_validate: bool, default_path: str = "/opt/snare") -> None: - """Base class for all core functions of the cloner + """Constructor method :param root: Website root URL :type root: str @@ -109,7 +111,7 @@ def get_headers( return headers, content_type async def process_link(self, url: str, level: int, check_host: bool = False) -> Union[str, None]: - """Process (relative and absolute) links to make them suitable for serving and add new URLs to the queue + """Add URL to the queue if new and return its relative URL :param url: Page URL :type url: str @@ -152,7 +154,7 @@ async def process_link(self, url: str, level: int, check_host: bool = False) -> return res async def replace_links(self, data: Union[bytes, str], level: int) -> BeautifulSoup: - """Replace website links to make them suitable for serving + """Replace all links present in the page's data with their relative versions :param data: Page data :type data: Union[bytes, str] @@ -299,10 +301,12 @@ async def get_root_host(self) -> None: class SimpleCloner(BaseCloner): + """aiohttp-driven data fetching""" + async def fetch_data( self, session: aiohttp.ClientSession, current_url: yarl.URL, level: int, try_count: int ) -> Tuple[Union[yarl.URL, None], bytes, List[Dict[str, str]], str]: - """Fetch data from the given URL using aiohttp + """Fetch data from the given URL using aiohttp's ClientSession :param session: aiohttp ClientSession object :type session: aiohttp.ClientSession @@ -335,10 +339,12 @@ async def fetch_data( class HeadlessCloner(BaseCloner): + """Pyppeteer-driven data fetching""" + async def fetch_data( self, browser: pyppeteer.browser.Browser, current_url: yarl.URL, level: int, try_count: int ) -> Tuple[Union[yarl.URL, None], Union[str, bytes], List[Dict[str, str]], Union[str, None]]: - """Fetch data from the given URL using Pyppeteer + """Fetch data from the given URL using Pyppeteer's headless browser :param browser: Pyppeteer Browser object :type browser: pyppeteer.Browser.browser @@ -378,10 +384,12 @@ async def fetch_data( class CloneRunner: + """One class to rule them all - Runner class for all cloners""" + def __init__( self, root: str, max_depth: int, css_validate: bool, default_path: str = "/opt/snare", headless: bool = False ) -> None: - """Runner class for all cloners + """Constructor method :param root: Website root URL :type root: str diff --git a/snare/html_handler.py b/snare/html_handler.py index 8c8551a1..111f4fed 100644 --- a/snare/html_handler.py +++ b/snare/html_handler.py @@ -9,8 +9,10 @@ class HtmlHandler: + """Handle HTML content of pages""" + def __init__(self, no_dorks: bool, tanner: str): - """Class to handle HTML contents of pages + """Constructor method :param no_dorks: Disable dorks :type no_dorks: bool diff --git a/snare/middlewares.py b/snare/middlewares.py index 9b7fc74d..1ddf9372 100644 --- a/snare/middlewares.py +++ b/snare/middlewares.py @@ -6,10 +6,12 @@ class SnareMiddleware: + """Middleware for Snare's aiohttp web server""" + def __init__( self, error_404: Union[None, str], headers: List[Dict[str, str]] = [], server_header: str = "" ) -> None: - """Middleware class for Snare's aiohttp web server + """Constructor method :param error_404: 404 page's file name (hash) :type error_404: Union[None, str] diff --git a/snare/server.py b/snare/server.py index 871beb75..c7132128 100644 --- a/snare/server.py +++ b/snare/server.py @@ -13,6 +13,8 @@ class HttpRequestHandler: + """Handle all HTTP requests to Snare""" + def __init__( self, meta: Dict, @@ -22,7 +24,7 @@ def __init__( keep_alive: int = 75, **kwargs: Dict[str, str] ) -> None: - """HTTP request handler class + """Constructor method :param meta: Meta info from `meta.json` :type meta: Dict diff --git a/snare/tanner_handler.py b/snare/tanner_handler.py index eaf1fc70..8b3998fe 100644 --- a/snare/tanner_handler.py +++ b/snare/tanner_handler.py @@ -15,8 +15,10 @@ class TannerHandler: + """Handle Tanner communication""" + def __init__(self, run_args: argparse.Namespace, meta: Dict, snare_uuid: bytes) -> None: - """Class for handling Tanner communication + """Constructor method :param run_args: Runtime CLI arguments :type run_args: argparse.Namespace diff --git a/snare/utils/asyncmock.py b/snare/utils/asyncmock.py index 82e07310..2feac683 100644 --- a/snare/utils/asyncmock.py +++ b/snare/utils/asyncmock.py @@ -1,7 +1,9 @@ from unittest.mock import Mock -class AsyncMock(Mock): # custom function defined to mock asyncio coroutines +class AsyncMock(Mock): + """Custom class defined to mock asyncio coroutines""" + def __call__(self, *args, **kwargs): sup = super(AsyncMock, self) diff --git a/snare/utils/logger.py b/snare/utils/logger.py index 0adb9880..bd071b6a 100644 --- a/snare/utils/logger.py +++ b/snare/utils/logger.py @@ -21,12 +21,13 @@ def filter(self, record: logging.LogRecord) -> bool: :return: True if record's level is lesser than the set level :rtype: bool """ + # "<" instead of "<=": since logger.setLevel is inclusive, this should be exclusive return record.levelno < self.level - # "<" instead of "<=": since logger.setLevel is inclusive, this should be exclusive - class Logger: + """Modify built-in logger's format and handlers for Snare and Cloner""" + @staticmethod def create_logger(debug_filename: str, err_filename: str, logger_name: str) -> logging.Logger: """Create logger with debugging and error level handlers for Snare diff --git a/snare/utils/snare_helpers.py b/snare/utils/snare_helpers.py index bb4d1535..6a7f1c7c 100755 --- a/snare/utils/snare_helpers.py +++ b/snare/utils/snare_helpers.py @@ -11,8 +11,10 @@ class VersionManager: + """Check Snare-Tanner compatibility""" + def __init__(self) -> None: - """Version manager class for Snare-Tanner compatibility checking""" + """Constructor method""" self.logger = logging.getLogger(__name__) self.version = "0.3.0" self.version_mapper = { @@ -40,13 +42,15 @@ def check_compatibility(self, tanner_version: str) -> None: class Converter: + """Convert a website's source files to a Snare-friendly form""" + def __init__(self) -> None: - """Converter class""" + """Constructor method""" self.logger = logging.getLogger(__name__) self.meta = {} def convert(self, path: str) -> None: - """Convert all pages to a Snare-friendly form and write meta info + """Rename all page files to their MD5 hash and populate meta.json with their hash and Content-Type header :param path: Page files storage directory :type path: str