From 4b98f17ce755b7384182441fd5b4fef45c960c64 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 27 Apr 2023 01:00:31 +0000 Subject: [PATCH 01/38] Bump requests from 2.28.2 to 2.29.0 Bumps [requests](https://github.com/psf/requests) from 2.28.2 to 2.29.0. - [Release notes](https://github.com/psf/requests/releases) - [Changelog](https://github.com/psf/requests/blob/main/HISTORY.md) - [Commits](https://github.com/psf/requests/compare/v2.28.2...v2.29.0) --- updated-dependencies: - dependency-name: requests dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 7e5d5dc7..7c1a61c4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ flake8==6.0.0 pre-commit==3.2.2 qbittorrent-api==2023.4.47 -requests==2.28.2 +requests==2.29.0 retrying==1.3.4 ruamel.yaml==0.17.21 schedule==1.2.0 From b693b97fb0394b45f8fe35afe53a58f5bcb2141c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 2 May 2023 00:59:58 +0000 Subject: [PATCH 02/38] Bump pre-commit from 3.2.2 to 3.3.0 Bumps [pre-commit](https://github.com/pre-commit/pre-commit) from 3.2.2 to 3.3.0. - [Release notes](https://github.com/pre-commit/pre-commit/releases) - [Changelog](https://github.com/pre-commit/pre-commit/blob/main/CHANGELOG.md) - [Commits](https://github.com/pre-commit/pre-commit/compare/v3.2.2...v3.3.0) --- updated-dependencies: - dependency-name: pre-commit dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 7c1a61c4..b9c9bc57 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ flake8==6.0.0 -pre-commit==3.2.2 +pre-commit==3.3.0 qbittorrent-api==2023.4.47 requests==2.29.0 retrying==1.3.4 From 02c84078ee9db87c347959d16101fdb1f995bc0f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 3 May 2023 00:59:37 +0000 Subject: [PATCH 03/38] Bump pre-commit from 3.3.0 to 3.3.1 Bumps [pre-commit](https://github.com/pre-commit/pre-commit) from 3.3.0 to 3.3.1. - [Release notes](https://github.com/pre-commit/pre-commit/releases) - [Changelog](https://github.com/pre-commit/pre-commit/blob/main/CHANGELOG.md) - [Commits](https://github.com/pre-commit/pre-commit/compare/v3.3.0...v3.3.1) --- updated-dependencies: - dependency-name: pre-commit dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b9c9bc57..4af11516 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ flake8==6.0.0 -pre-commit==3.3.0 +pre-commit==3.3.1 qbittorrent-api==2023.4.47 requests==2.29.0 retrying==1.3.4 From 963b0109b37f29206458c12dc1e63716e516b64a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 3 May 2023 00:59:50 +0000 Subject: [PATCH 04/38] Bump ruamel-yaml from 0.17.21 to 0.17.22 Bumps [ruamel-yaml](https://sourceforge.net/p/ruamel-yaml/code/ci/default/tree) from 0.17.21 to 0.17.22. --- updated-dependencies: - dependency-name: ruamel-yaml dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b9c9bc57..3335c386 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,5 +3,5 @@ pre-commit==3.3.0 qbittorrent-api==2023.4.47 requests==2.29.0 retrying==1.3.4 -ruamel.yaml==0.17.21 +ruamel.yaml==0.17.22 schedule==1.2.0 From 007986ba5bb681312bbe90a7b54388562578ca84 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 5 May 2023 01:01:36 +0000 Subject: [PATCH 05/38] Bump requests from 2.29.0 to 2.30.0 Bumps [requests](https://github.com/psf/requests) from 2.29.0 to 2.30.0. - [Release notes](https://github.com/psf/requests/releases) - [Changelog](https://github.com/psf/requests/blob/main/HISTORY.md) - [Commits](https://github.com/psf/requests/compare/v2.29.0...v2.30.0) --- updated-dependencies: - dependency-name: requests dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index af567357..653cd2a6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ flake8==6.0.0 pre-commit==3.3.1 qbittorrent-api==2023.4.47 -requests==2.29.0 +requests==2.30.0 retrying==1.3.4 ruamel.yaml==0.17.22 schedule==1.2.0 From 3a8bdf526ff4be55b49e798bdc9696688c6ef4b5 Mon Sep 17 00:00:00 2001 From: bobokun Date: Sat, 6 May 2023 09:37:11 -0400 Subject: [PATCH 06/38] Additional error checking to fix #282 --- modules/config.py | 43 ++++++++++++++++++++++--------------------- qbit_manage.py | 14 +++++--------- 2 files changed, 27 insertions(+), 30 deletions(-) diff --git a/modules/config.py b/modules/config.py index f884dc62..b09a0702 100755 --- a/modules/config.py +++ b/modules/config.py @@ -110,27 +110,28 @@ def __init__(self, default_dir, args): self.data["notifiarr"] = self.data.pop("notifiarr") if "webhooks" in self.data: temp = self.data.pop("webhooks") - if "function" not in temp or ("function" in temp and temp["function"] is None): - temp["function"] = {} - - def hooks(attr): - if attr in temp: - items = temp.pop(attr) - if items: - temp["function"][attr] = items - if attr not in temp["function"]: - temp["function"][attr] = {} - temp["function"][attr] = None - - hooks("cross_seed") - hooks("recheck") - hooks("cat_update") - hooks("tag_update") - hooks("rem_unregistered") - hooks("rem_orphaned") - hooks("tag_nohardlinks") - hooks("cleanup_dirs") - self.data["webhooks"] = temp + if temp is not None: + if "function" not in temp or ("function" in temp and temp["function"] is None): + temp["function"] = {} + + def hooks(attr): + if attr in temp: + items = temp.pop(attr) + if items: + temp["function"][attr] = items + if attr not in temp["function"]: + temp["function"][attr] = {} + temp["function"][attr] = None + + hooks("cross_seed") + hooks("recheck") + hooks("cat_update") + hooks("tag_update") + hooks("rem_unregistered") + hooks("rem_orphaned") + hooks("tag_nohardlinks") + hooks("cleanup_dirs") + self.data["webhooks"] = temp if "bhd" in self.data: self.data["bhd"] = self.data.pop("bhd") self.dry_run = self.commands["dry_run"] diff --git a/qbit_manage.py b/qbit_manage.py index 0ca6f61b..0250a757 100755 --- a/qbit_manage.py +++ b/qbit_manage.py @@ -377,16 +377,12 @@ def finished_run(): try: cfg = Config(default_dir, args) qbit_manager = cfg.qbt - except Exception as ex: - if "Qbittorrent Error" in ex.args[0]: - logger.print_line(ex, "CRITICAL") - logger.print_line("Exiting scheduled Run.", "CRITICAL") - finished_run() - return None - else: - logger.stacktrace() - logger.print_line(ex, "CRITICAL") + logger.stacktrace() + logger.print_line(ex, "CRITICAL") + logger.print_line("Exiting scheduled Run.", "CRITICAL") + finished_run() + return None if qbit_manager: # Set Category From 4bca6f142af284a1e56f67dfb5ad9b8de569a899 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 May 2023 01:06:58 +0000 Subject: [PATCH 07/38] Bump ruamel-yaml from 0.17.22 to 0.17.24 Bumps [ruamel-yaml](https://sourceforge.net/p/ruamel-yaml/code/ci/default/tree) from 0.17.22 to 0.17.24. --- updated-dependencies: - dependency-name: ruamel-yaml dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 653cd2a6..119172b3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,5 +3,5 @@ pre-commit==3.3.1 qbittorrent-api==2023.4.47 requests==2.30.0 retrying==1.3.4 -ruamel.yaml==0.17.22 +ruamel.yaml==0.17.24 schedule==1.2.0 From 50a88759c6cf15f24dc5a67657bf1cc8022d39ce Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 10 May 2023 01:12:29 +0000 Subject: [PATCH 08/38] Bump ruamel-yaml from 0.17.24 to 0.17.26 Bumps [ruamel-yaml](https://sourceforge.net/p/ruamel-yaml/code/ci/default/tree) from 0.17.24 to 0.17.26. --- updated-dependencies: - dependency-name: ruamel-yaml dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 119172b3..b34add45 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,5 +3,5 @@ pre-commit==3.3.1 qbittorrent-api==2023.4.47 requests==2.30.0 retrying==1.3.4 -ruamel.yaml==0.17.24 +ruamel.yaml==0.17.26 schedule==1.2.0 From eeb87f09486800bee3b13e9842749c19148a9e56 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 18 May 2023 01:00:24 +0000 Subject: [PATCH 09/38] Bump pre-commit from 3.3.1 to 3.3.2 Bumps [pre-commit](https://github.com/pre-commit/pre-commit) from 3.3.1 to 3.3.2. - [Release notes](https://github.com/pre-commit/pre-commit/releases) - [Changelog](https://github.com/pre-commit/pre-commit/blob/main/CHANGELOG.md) - [Commits](https://github.com/pre-commit/pre-commit/compare/v3.3.1...v3.3.2) --- updated-dependencies: - dependency-name: pre-commit dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b34add45..3df979a8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ flake8==6.0.0 -pre-commit==3.3.1 +pre-commit==3.3.2 qbittorrent-api==2023.4.47 requests==2.30.0 retrying==1.3.4 From 294292eb5872687a167ce9dc6357d485dc18d3f4 Mon Sep 17 00:00:00 2001 From: buthed010203 Date: Thu, 18 May 2023 14:04:17 -0400 Subject: [PATCH 10/38] Fix remove_empty_directories crashing when run in parallel --- modules/util.py | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/modules/util.py b/modules/util.py index 46dfe235..ad7d7347 100755 --- a/modules/util.py +++ b/modules/util.py @@ -307,19 +307,22 @@ def copy_files(src, dest): def remove_empty_directories(pathlib_root_dir, pattern): """Remove empty directories recursively.""" pathlib_root_dir = Path(pathlib_root_dir) - # list all directories recursively and sort them by path, - # longest first - longest = sorted( - pathlib_root_dir.glob(pattern), - key=lambda p: len(str(p)), - reverse=True, - ) - longest.append(pathlib_root_dir) - for pdir in longest: - try: - pdir.rmdir() # remove directory if empty - except OSError: - continue # catch and continue if non-empty + try: + # list all directories recursively and sort them by path, + # longest first + longest = sorted( + pathlib_root_dir.glob(pattern), + key=lambda p: len(str(p)), + reverse=True, + ) + longest.append(pathlib_root_dir) # delete the folder itself if it's empty + for pdir in longest: + try: + pdir.rmdir() # remove directory if empty + except (FileNotFoundError, OSError): + continue # catch and continue if non-empty, folders within could already be deleted if run in parallel + except FileNotFoundError: + pass # if this is being run in parallel, pathlib_root_dir could already be deleted def nohardlink(file, notify): From cc723d529df7734fd2ff6565ea53d7ab889814fe Mon Sep 17 00:00:00 2001 From: buthed010203 Date: Thu, 18 May 2023 14:15:54 -0400 Subject: [PATCH 11/38] Fix crash in cross-seed module --- modules/core/cross_seed.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/core/cross_seed.py b/modules/core/cross_seed.py index 80e79244..61f10b03 100644 --- a/modules/core/cross_seed.py +++ b/modules/core/cross_seed.py @@ -72,7 +72,7 @@ def cross_seed(self): logger.print_line(f"Found {t_name} in {dir_cs} but original torrent is not complete.", self.config.loglevel) logger.print_line("Not adding to qBittorrent", self.config.loglevel) else: - error = f"{t_name} not found in torrents. Cross-seed Torrent not added to qBittorrent." + error = f"{tr_name} not found in torrents. Cross-seed Torrent not added to qBittorrent." if self.config.dry_run: logger.print_line(error, self.config.loglevel) else: From e606a7d579d424ac517df120039425fe16518567 Mon Sep 17 00:00:00 2001 From: bobokun Date: Fri, 19 May 2023 21:45:14 -0400 Subject: [PATCH 12/38] 3.6.3 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index b7276283..4a788a01 100755 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.6.2 +3.6.3 From 1ed99aeb3e1f3ee7f984c2f2e6e5919d3b76d52c Mon Sep 17 00:00:00 2001 From: bobokun Date: Fri, 19 May 2023 21:52:06 -0400 Subject: [PATCH 13/38] Changes HardLink Detection logic (fixes #291) --- README.md | 2 +- config/config.yml.sample | 2 +- modules/core/remove_orphaned.py | 16 +---- modules/core/tag_nohardlinks.py | 3 +- modules/util.py | 104 +++++++++++++++++++++----------- 5 files changed, 74 insertions(+), 53 deletions(-) diff --git a/README.md b/README.md index bf54f823..356ceae9 100755 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ This is a program used to manage your qBittorrent instance such as: * Automatically add [cross-seed](https://github.com/mmgoodnow/cross-seed) torrents in paused state. **\*Note: cross-seed now allows for torrent injections directly to qBit, making this feature obsolete.\*** * Recheck paused torrents sorted by lowest size and resume if completed * Remove orphaned files from your root directory that are not referenced by qBittorrent -* Tag any torrents that have no hard links and allows optional cleanup to delete these torrents and contents based on maximum ratio and/or time seeded +* Tag any torrents that have no hard links outisde the root folder and allows optional cleanup to delete these torrents and contents based on maximum ratio and/or time seeded * RecycleBin function to move files into a RecycleBin folder instead of deleting the data directly when deleting a torrent * Built-in scheduler to run the script every x minutes. (Can use `--run` command to run without the scheduler) * Webhook notifications with [Notifiarr](https://notifiarr.com/) and [Apprise API](https://github.com/caronc/apprise-api) integration diff --git a/config/config.yml.sample b/config/config.yml.sample index 68950567..4eca5630 100755 --- a/config/config.yml.sample +++ b/config/config.yml.sample @@ -136,7 +136,7 @@ tracker: tag: other nohardlinks: - # Tag Movies/Series that are not hard linked + # Tag Movies/Series that are not hard linked outside the root directory # Mandatory to fill out directory parameter above to use this function (root_dir/remote_dir) # This variable should be set to your category name of your completed movies/completed series in qbit. Acceptable variable can be any category you would like to tag if there are no hardlinks found movies-completed: diff --git a/modules/core/remove_orphaned.py b/modules/core/remove_orphaned.py index 0d6f4a49..030ee3b4 100644 --- a/modules/core/remove_orphaned.py +++ b/modules/core/remove_orphaned.py @@ -36,21 +36,7 @@ def rem_orphaned(self): orphaned_files = [] excluded_orphan_files = [] - if self.remote_dir != self.root_dir: - local_orphaned_dir = self.orphaned_dir.replace(self.remote_dir, self.root_dir) - root_files = [ - os.path.join(path.replace(self.remote_dir, self.root_dir), name) - for path, subdirs, files in os.walk(self.remote_dir) - for name in files - if local_orphaned_dir not in path - ] - else: - root_files = [ - os.path.join(path, name) - for path, subdirs, files in os.walk(self.root_dir) - for name in files - if self.orphaned_dir not in path - ] + root_files = util.get_root_files(self.remote_dir, self.root_dir, self.orphaned_dir) # Get an updated list of torrents logger.print_line("Locating orphan files", self.config.loglevel) diff --git a/modules/core/tag_nohardlinks.py b/modules/core/tag_nohardlinks.py index dbd3c8d3..97d4495c 100644 --- a/modules/core/tag_nohardlinks.py +++ b/modules/core/tag_nohardlinks.py @@ -186,6 +186,7 @@ def tag_nohardlinks(self): """Tag torrents with no hardlinks""" logger.separator("Tagging Torrents with No Hardlinks", space=False, border=False) nohardlinks = self.nohardlinks + check_hardlinks = util.CheckHardLinks(self.root_dir, self.remote_dir) for category in nohardlinks: torrent_list = self.qbt.get_torrents({"category": category, "status_filter": "completed"}) if len(torrent_list) == 0: @@ -199,7 +200,7 @@ def tag_nohardlinks(self): continue for torrent in torrent_list: tracker = self.qbt.get_tags(torrent.trackers) - has_nohardlinks = util.nohardlink( + has_nohardlinks = check_hardlinks.nohardlink( torrent["content_path"].replace(self.root_dir, self.remote_dir), self.config.notify ) if any(tag in torrent.tags for tag in nohardlinks[category]["exclude_tags"]): diff --git a/modules/util.py b/modules/util.py index ad7d7347..61c89aa4 100755 --- a/modules/util.py +++ b/modules/util.py @@ -325,44 +325,78 @@ def remove_empty_directories(pathlib_root_dir, pattern): pass # if this is being run in parallel, pathlib_root_dir could already be deleted -def nohardlink(file, notify): +class CheckHardLinks: """ - Check if there are any hard links - Will check if there are any hard links if it passes a file or folder - If a folder is passed, it will take the largest file in that folder and only check for hardlinks - of the remaining files where the file is greater size a percentage of the largest file - This fixes the bug in #192 + Class to check for hardlinks """ - check_for_hl = True - if os.path.isfile(file): - logger.trace(f"Checking file: {file}") - if os.stat(file).st_nlink > 1: - check_for_hl = False - else: - sorted_files = sorted(Path(file).rglob("*"), key=lambda x: os.stat(x).st_size, reverse=True) - logger.trace(f"Folder: {file}") - logger.trace(f"Files Sorted by size: {sorted_files}") - threshold = 0.5 - if not sorted_files: - msg = ( - f"Nohardlink Error: Unable to open the folder {file}. " - "Please make sure folder exists and qbit_manage has access to this directory." - ) - notify(msg, "nohardlink") - logger.warning(msg) + + def __init__(self, root_dir, remote_dir): + self.root_dir = root_dir + self.remote_dir = remote_dir + self.root_files = set(get_root_files(self.root_dir, self.remote_dir)) + self.get_inode_count() + + def get_inode_count(self): + self.inode_count = {} + for file in self.root_files: + inode_no = os.stat(file.replace(self.root_dir, self.remote_dir)).st_ino + if inode_no in self.inode_count: + self.inode_count[inode_no] += 1 + else: + self.inode_count[inode_no] = 1 + + def nohardlink(self, file, notify): + """ + Check if there are any hard links + Will check if there are any hard links if it passes a file or folder + If a folder is passed, it will take the largest file in that folder and only check for hardlinks + of the remaining files where the file is greater size a percentage of the largest file + This fixes the bug in #192 + """ + check_for_hl = True + if os.path.isfile(file): + logger.trace(f"Checking file: {file}") + # https://github.com/StuffAnThings/qbit_manage/issues/291 for more details + if os.stat(file).st_nlink - self.inode_count.get(os.stat(file).st_ino, 1) > 0: + check_for_hl = False else: - largest_file_size = os.stat(sorted_files[0]).st_size - logger.trace(f"Largest file: {sorted_files[0]}") - logger.trace(f"Largest file size: {largest_file_size}") - for files in sorted_files: - file_size = os.stat(files).st_size - file_no_hardlinks = os.stat(files).st_nlink - logger.trace(f"Checking file: {file}") - logger.trace(f"Checking file size: {file_size}") - logger.trace(f"Checking no of hard links: {file_no_hardlinks}") - if file_no_hardlinks > 1 and file_size >= (largest_file_size * threshold): - check_for_hl = False - return check_for_hl + sorted_files = sorted(Path(file).rglob("*"), key=lambda x: os.stat(x).st_size, reverse=True) + logger.trace(f"Folder: {file}") + logger.trace(f"Files Sorted by size: {sorted_files}") + threshold = 0.5 + if not sorted_files: + msg = ( + f"Nohardlink Error: Unable to open the folder {file}. " + "Please make sure folder exists and qbit_manage has access to this directory." + ) + notify(msg, "nohardlink") + logger.warning(msg) + else: + largest_file_size = os.stat(sorted_files[0]).st_size + logger.trace(f"Largest file: {sorted_files[0]}") + logger.trace(f"Largest file size: {largest_file_size}") + for files in sorted_files: + file_size = os.stat(files).st_size + file_no_hardlinks = os.stat(files).st_nlink + logger.trace(f"Checking file: {file}") + logger.trace(f"Checking file size: {file_size}") + logger.trace(f"Checking no of hard links: {file_no_hardlinks}") + if file_no_hardlinks - self.inode_count.get(os.stat(file).st_ino, 1) > 0 and file_size >= ( + largest_file_size * threshold + ): + check_for_hl = False + return check_for_hl + + +def get_root_files(root_dir, remote_dir, exclude_dir=None): + local_exclude_dir = exclude_dir.replace(remote_dir, root_dir) if exclude_dir and remote_dir != root_dir else exclude_dir + root_files = [ + os.path.join(path.replace(remote_dir, root_dir) if remote_dir != root_dir else path, name) + for path, subdirs, files in os.walk(remote_dir if remote_dir != root_dir else root_dir) + for name in files + if not local_exclude_dir or local_exclude_dir not in path + ] + return root_files def load_json(file): From 07b7350b60e3a0713e47fce0539b372bb2d1e182 Mon Sep 17 00:00:00 2001 From: bobokun Date: Sat, 20 May 2023 07:36:37 -0400 Subject: [PATCH 14/38] add additional tracing for noHL --- modules/util.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/modules/util.py b/modules/util.py index 61c89aa4..9c9bbe05 100755 --- a/modules/util.py +++ b/modules/util.py @@ -356,6 +356,9 @@ def nohardlink(self, file, notify): check_for_hl = True if os.path.isfile(file): logger.trace(f"Checking file: {file}") + logger.trace(f"Checking file inum: {os.stat(file).st_ino}") + logger.trace(f"Checking no of hard links: {os.stat(file).st_nlink}") + logger.tract(f"Checking inode_count dict: {self.inode_count.get(os.stat(file).st_ino)}") # https://github.com/StuffAnThings/qbit_manage/issues/291 for more details if os.stat(file).st_nlink - self.inode_count.get(os.stat(file).st_ino, 1) > 0: check_for_hl = False @@ -379,8 +382,10 @@ def nohardlink(self, file, notify): file_size = os.stat(files).st_size file_no_hardlinks = os.stat(files).st_nlink logger.trace(f"Checking file: {file}") + logger.trace(f"Checking file inum: {os.stat(file).st_ino}") logger.trace(f"Checking file size: {file_size}") logger.trace(f"Checking no of hard links: {file_no_hardlinks}") + logger.tract(f"Checking inode_count dict: {self.inode_count.get(os.stat(file).st_ino)}") if file_no_hardlinks - self.inode_count.get(os.stat(file).st_ino, 1) > 0 and file_size >= ( largest_file_size * threshold ): From e07c3e417badc14e99f6e580d53e81c51b244e05 Mon Sep 17 00:00:00 2001 From: bobokun Date: Sat, 20 May 2023 11:19:30 -0400 Subject: [PATCH 15/38] return False by default when there are permission issues in noHL --- modules/util.py | 81 ++++++++++++++++++++++++++++--------------------- 1 file changed, 46 insertions(+), 35 deletions(-) diff --git a/modules/util.py b/modules/util.py index 9c9bbe05..859d62ac 100755 --- a/modules/util.py +++ b/modules/util.py @@ -354,42 +354,53 @@ def nohardlink(self, file, notify): This fixes the bug in #192 """ check_for_hl = True - if os.path.isfile(file): - logger.trace(f"Checking file: {file}") - logger.trace(f"Checking file inum: {os.stat(file).st_ino}") - logger.trace(f"Checking no of hard links: {os.stat(file).st_nlink}") - logger.tract(f"Checking inode_count dict: {self.inode_count.get(os.stat(file).st_ino)}") - # https://github.com/StuffAnThings/qbit_manage/issues/291 for more details - if os.stat(file).st_nlink - self.inode_count.get(os.stat(file).st_ino, 1) > 0: - check_for_hl = False - else: - sorted_files = sorted(Path(file).rglob("*"), key=lambda x: os.stat(x).st_size, reverse=True) - logger.trace(f"Folder: {file}") - logger.trace(f"Files Sorted by size: {sorted_files}") - threshold = 0.5 - if not sorted_files: - msg = ( - f"Nohardlink Error: Unable to open the folder {file}. " - "Please make sure folder exists and qbit_manage has access to this directory." - ) - notify(msg, "nohardlink") - logger.warning(msg) + try: + if os.path.isfile(file): + logger.trace(f"Checking file: {file}") + logger.trace(f"Checking file inum: {os.stat(file).st_ino}") + logger.trace(f"Checking no of hard links: {os.stat(file).st_nlink}") + logger.tract(f"Checking inode_count dict: {self.inode_count.get(os.stat(file).st_ino)}") + # https://github.com/StuffAnThings/qbit_manage/issues/291 for more details + if os.stat(file).st_nlink - self.inode_count.get(os.stat(file).st_ino, 1) > 0: + check_for_hl = False else: - largest_file_size = os.stat(sorted_files[0]).st_size - logger.trace(f"Largest file: {sorted_files[0]}") - logger.trace(f"Largest file size: {largest_file_size}") - for files in sorted_files: - file_size = os.stat(files).st_size - file_no_hardlinks = os.stat(files).st_nlink - logger.trace(f"Checking file: {file}") - logger.trace(f"Checking file inum: {os.stat(file).st_ino}") - logger.trace(f"Checking file size: {file_size}") - logger.trace(f"Checking no of hard links: {file_no_hardlinks}") - logger.tract(f"Checking inode_count dict: {self.inode_count.get(os.stat(file).st_ino)}") - if file_no_hardlinks - self.inode_count.get(os.stat(file).st_ino, 1) > 0 and file_size >= ( - largest_file_size * threshold - ): - check_for_hl = False + sorted_files = sorted(Path(file).rglob("*"), key=lambda x: os.stat(x).st_size, reverse=True) + logger.trace(f"Folder: {file}") + logger.trace(f"Files Sorted by size: {sorted_files}") + threshold = 0.5 + if not sorted_files: + msg = ( + f"Nohardlink Error: Unable to open the folder {file}. " + "Please make sure folder exists and qbit_manage has access to this directory." + ) + notify(msg, "nohardlink") + logger.warning(msg) + else: + largest_file_size = os.stat(sorted_files[0]).st_size + logger.trace(f"Largest file: {sorted_files[0]}") + logger.trace(f"Largest file size: {largest_file_size}") + for files in sorted_files: + file_size = os.stat(files).st_size + file_no_hardlinks = os.stat(files).st_nlink + logger.trace(f"Checking file: {file}") + logger.trace(f"Checking file inum: {os.stat(file).st_ino}") + logger.trace(f"Checking file size: {file_size}") + logger.trace(f"Checking no of hard links: {file_no_hardlinks}") + logger.tract(f"Checking inode_count dict: {self.inode_count.get(os.stat(file).st_ino)}") + if file_no_hardlinks - self.inode_count.get(os.stat(file).st_ino, 1) > 0 and file_size >= ( + largest_file_size * threshold + ): + check_for_hl = False + except PermissionError as perm: + logger.warning(f"{perm} : file {file} has permission issues.") + return False + except FileNotFoundError as file_not_found_error: + logger.warning(f"{file_not_found_error} : File {file} not found.") + return False + except Exception as ex: + logger.stacktrace() + logger.error(ex) + return False return check_for_hl From 4e234de7f6537ccc88ecc272e8f6faa21a384b76 Mon Sep 17 00:00:00 2001 From: bobokun Date: Sat, 20 May 2023 11:35:27 -0400 Subject: [PATCH 16/38] Default skip symlinks in noHL --- modules/util.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/modules/util.py b/modules/util.py index 859d62ac..89f9b3be 100755 --- a/modules/util.py +++ b/modules/util.py @@ -356,10 +356,13 @@ def nohardlink(self, file, notify): check_for_hl = True try: if os.path.isfile(file): + if os.path.islink(file): + logger.warning(f"Symlink found in {file}, unable to determine hardlinks. Skipping...") + return False logger.trace(f"Checking file: {file}") logger.trace(f"Checking file inum: {os.stat(file).st_ino}") logger.trace(f"Checking no of hard links: {os.stat(file).st_nlink}") - logger.tract(f"Checking inode_count dict: {self.inode_count.get(os.stat(file).st_ino)}") + logger.trace(f"Checking inode_count dict: {self.inode_count.get(os.stat(file).st_ino)}") # https://github.com/StuffAnThings/qbit_manage/issues/291 for more details if os.stat(file).st_nlink - self.inode_count.get(os.stat(file).st_ino, 1) > 0: check_for_hl = False @@ -380,6 +383,9 @@ def nohardlink(self, file, notify): logger.trace(f"Largest file: {sorted_files[0]}") logger.trace(f"Largest file size: {largest_file_size}") for files in sorted_files: + if os.path.islink(files): + logger.warning(f"Symlink found in {files}, unable to determine hardlinks. Skipping...") + continue file_size = os.stat(files).st_size file_no_hardlinks = os.stat(files).st_nlink logger.trace(f"Checking file: {file}") @@ -392,10 +398,10 @@ def nohardlink(self, file, notify): ): check_for_hl = False except PermissionError as perm: - logger.warning(f"{perm} : file {file} has permission issues.") + logger.warning(f"{perm} : file {file} has permission issues. Skipping...") return False except FileNotFoundError as file_not_found_error: - logger.warning(f"{file_not_found_error} : File {file} not found.") + logger.warning(f"{file_not_found_error} : File {file} not found. Skipping...") return False except Exception as ex: logger.stacktrace() From 6ddeca80d4c3fb4d16df8dfd347be85d2a6b6225 Mon Sep 17 00:00:00 2001 From: bobokun Date: Sat, 20 May 2023 11:38:43 -0400 Subject: [PATCH 17/38] typo --- modules/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/util.py b/modules/util.py index 89f9b3be..c4734b4e 100755 --- a/modules/util.py +++ b/modules/util.py @@ -392,7 +392,7 @@ def nohardlink(self, file, notify): logger.trace(f"Checking file inum: {os.stat(file).st_ino}") logger.trace(f"Checking file size: {file_size}") logger.trace(f"Checking no of hard links: {file_no_hardlinks}") - logger.tract(f"Checking inode_count dict: {self.inode_count.get(os.stat(file).st_ino)}") + logger.trace(f"Checking inode_count dict: {self.inode_count.get(os.stat(file).st_ino)}") if file_no_hardlinks - self.inode_count.get(os.stat(file).st_ino, 1) > 0 and file_size >= ( largest_file_size * threshold ): From e817472ccdabe571bfb4b51661774695f6c2fb5c Mon Sep 17 00:00:00 2001 From: buthed010203 Date: Sat, 20 May 2023 12:00:38 -0400 Subject: [PATCH 18/38] Potential fix for windows crashing on RemoveOrphaned --- modules/core/remove_orphaned.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/modules/core/remove_orphaned.py b/modules/core/remove_orphaned.py index 030ee3b4..202bec6f 100644 --- a/modules/core/remove_orphaned.py +++ b/modules/core/remove_orphaned.py @@ -7,7 +7,6 @@ from modules import util logger = util.logger -_config = None class RemoveOrphaned: @@ -23,7 +22,7 @@ def __init__(self, qbit_manager): global _config _config = self.config - self.pool = Pool(processes=max(cpu_count() - 1, 1)) + self.pool = Pool(processes=max(cpu_count() - 1, 1), initializer=init_pool, initargs=(self.config,)) self.rem_orphaned() self.cleanup_pool() @@ -106,6 +105,10 @@ def cleanup_pool(self): self.pool.join() +def init_pool(conf): + global _config + _config = conf + def get_full_path_of_torrent_files(torrent_files, save_path): fullpath_torrent_files = [] for file in torrent_files: @@ -115,7 +118,6 @@ def get_full_path_of_torrent_files(torrent_files, save_path): fullpath_torrent_files.append(fullpath) return fullpath_torrent_files - def move_orphan(file): src = file.replace(_config.root_dir, _config.remote_dir) # Could be optimized to only run when root != remote dest = os.path.join(_config.orphaned_dir, file.replace(_config.root_dir, "")) From e295bdbf4657ba4b7b7843e34b48fb16d32c982b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 20 May 2023 16:02:06 +0000 Subject: [PATCH 19/38] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- modules/core/remove_orphaned.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/core/remove_orphaned.py b/modules/core/remove_orphaned.py index 202bec6f..5a80754e 100644 --- a/modules/core/remove_orphaned.py +++ b/modules/core/remove_orphaned.py @@ -109,6 +109,7 @@ def init_pool(conf): global _config _config = conf + def get_full_path_of_torrent_files(torrent_files, save_path): fullpath_torrent_files = [] for file in torrent_files: @@ -118,6 +119,7 @@ def get_full_path_of_torrent_files(torrent_files, save_path): fullpath_torrent_files.append(fullpath) return fullpath_torrent_files + def move_orphan(file): src = file.replace(_config.root_dir, _config.remote_dir) # Could be optimized to only run when root != remote dest = os.path.join(_config.orphaned_dir, file.replace(_config.root_dir, "")) From 2c24abbcdf1c48caf316880a6170d7af141b4b5e Mon Sep 17 00:00:00 2001 From: buthed010203 Date: Sat, 20 May 2023 12:03:31 -0400 Subject: [PATCH 20/38] Remove redundant code --- modules/core/remove_orphaned.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/modules/core/remove_orphaned.py b/modules/core/remove_orphaned.py index 5a80754e..fdfd99e8 100644 --- a/modules/core/remove_orphaned.py +++ b/modules/core/remove_orphaned.py @@ -20,8 +20,6 @@ def __init__(self, qbit_manager): self.root_dir = qbit_manager.config.root_dir self.orphaned_dir = qbit_manager.config.orphaned_dir - global _config - _config = self.config self.pool = Pool(processes=max(cpu_count() - 1, 1), initializer=init_pool, initargs=(self.config,)) self.rem_orphaned() self.cleanup_pool() From 55c834997f5b5f62a2211cd69e925eb9f73f7179 Mon Sep 17 00:00:00 2001 From: bobokun Date: Sat, 20 May 2023 12:33:55 -0400 Subject: [PATCH 21/38] update PR template --- .github/pull_request_template.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 1899743b..6fd6563e 100755 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -26,4 +26,4 @@ Please delete options that are not relevant. - [ ] I have performed a self-review of my own code - [ ] I have commented my code, particularly in hard-to-understand areas - [ ] I have added or updated the docstring for new or existing methods -- [ ] I have added tests when applicable +- [ ] I have modified this PR to merge to the develop branch From 0cc01976b1497e56b49d79c145930037f1db3786 Mon Sep 17 00:00:00 2001 From: bobokun Date: Sat, 20 May 2023 16:23:28 -0400 Subject: [PATCH 22/38] Fixes #292 --- modules/core/cross_seed.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/core/cross_seed.py b/modules/core/cross_seed.py index 61f10b03..cbad7faa 100644 --- a/modules/core/cross_seed.py +++ b/modules/core/cross_seed.py @@ -39,7 +39,7 @@ def cross_seed(self): dest = os.path.join(self.qbt.torrentinfo[t_name]["save_path"], "") src = os.path.join(dir_cs, file) dir_cs_out = os.path.join(dir_cs, "qbit_manage_added", file) - category = self.qbt.get_category(dest) + category = self.qbt.torrentinfo[t_name].get("Category", self.qbt.get_category(dest)) # Only add cross-seed torrent if original torrent is complete if self.qbt.torrentinfo[t_name]["is_complete"]: categories.append(category) From 4e02ee4e91e295fb3b68d6099734db5ffb592b6a Mon Sep 17 00:00:00 2001 From: bobokun Date: Mon, 22 May 2023 10:08:08 -0400 Subject: [PATCH 23/38] updates logging --- modules/logs.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/modules/logs.py b/modules/logs.py index f580dfd4..ef007436 100755 --- a/modules/logs.py +++ b/modules/logs.py @@ -17,7 +17,7 @@ DRYRUN = 25 INFO = 20 DEBUG = 10 -TRACE = 5 +TRACE = 0 def fmt_filter(record): @@ -72,17 +72,19 @@ def _get_handler(self, log_file, count=3): """Get handler for log file""" max_bytes = 1024 * 1024 * 2 _handler = RotatingFileHandler(log_file, delay=True, mode="w", maxBytes=max_bytes, backupCount=count, encoding="utf-8") - self._formatter(_handler) + self._formatter(handler=_handler) # if os.path.isfile(log_file): # _handler.doRollover() return _handler - def _formatter(self, handler, border=True): + def _formatter(self, handler=None, border=True, log_only=False, space=False): """Format log message""" - text = f"| %(message)-{self.screen_width - 2}s |" if border else f"%(message)-{self.screen_width - 2}s" - if isinstance(handler, RotatingFileHandler): - text = f"[%(asctime)s] %(filename)-27s %(levelname)-10s {text}" - handler.setFormatter(logging.Formatter(text)) + console = f"| %(message)-{self.screen_width - 2}s |" if border else f"%(message)-{self.screen_width - 2}s" + file = f"{' '*65}" if space else "[%(asctime)s] %(filename)-27s %(levelname)-10s " + handlers = [handler] if handler else self._logger.handlers + for h in handlers: + if not log_only or isinstance(h, RotatingFileHandler): + h.setFormatter(logging.Formatter(f"{file if isinstance(h, RotatingFileHandler) else ''}{console}")) def add_main_handler(self): """Add main handler to logger""" @@ -233,18 +235,15 @@ def insert_space(self, display_title, space_length=0): def _log(self, level, msg, args, exc_info=None, extra=None, stack_info=False, stacklevel=1): """Log""" + log_only = False if self.spacing > 0: self.exorcise() if "\n" in msg: for i, line in enumerate(msg.split("\n")): self._log(level, line, args, exc_info=exc_info, extra=extra, stack_info=stack_info, stacklevel=stacklevel) if i == 0: - for handler in self._logger.handlers: - if isinstance(handler, RotatingFileHandler): - handler.setFormatter(logging.Formatter(" " * 65 + "| %(message)s")) - for handler in self._logger.handlers: - if isinstance(handler, RotatingFileHandler): - handler.setFormatter(logging.Formatter("[%(asctime)s] %(filename)-27s %(levelname)-10s | %(message)s")) + self._formatter(log_only=True, space=True) + log_only = True else: for secret in sorted(self.secrets, reverse=True): if secret in msg: @@ -266,6 +265,8 @@ def _log(self, level, msg, args, exc_info=None, extra=None, stack_info=False, st exc_info = sys.exc_info() record = self._logger.makeRecord(self._logger.name, level, func, lno, msg, args, exc_info, func, extra, sinfo) self._logger.handle(record) + if log_only: + self._formatter() def find_caller(self, stack_info=False, stacklevel=1): """Find caller""" From f8a6b1d5a483207fffe6a07559fa5d10c6799abf Mon Sep 17 00:00:00 2001 From: bobokun Date: Mon, 22 May 2023 10:43:31 -0400 Subject: [PATCH 24/38] Uses threading instead of multiprocessing Fixes #275 --- modules/core/remove_orphaned.py | 37 +++++++++++++-------------------- 1 file changed, 14 insertions(+), 23 deletions(-) diff --git a/modules/core/remove_orphaned.py b/modules/core/remove_orphaned.py index fdfd99e8..772ccc25 100644 --- a/modules/core/remove_orphaned.py +++ b/modules/core/remove_orphaned.py @@ -1,8 +1,7 @@ import os +from concurrent.futures import ThreadPoolExecutor from fnmatch import fnmatch from itertools import repeat -from multiprocessing import cpu_count -from multiprocessing import Pool from modules import util @@ -20,9 +19,10 @@ def __init__(self, qbit_manager): self.root_dir = qbit_manager.config.root_dir self.orphaned_dir = qbit_manager.config.orphaned_dir - self.pool = Pool(processes=max(cpu_count() - 1, 1), initializer=init_pool, initargs=(self.config,)) + max_workers = max(os.cpu_count() - 1, 1) + self.executor = ThreadPoolExecutor(max_workers=max_workers) self.rem_orphaned() - self.cleanup_pool() + self.executor.shutdown() def rem_orphaned(self): """Remove orphaned files from remote directory""" @@ -47,7 +47,7 @@ def rem_orphaned(self): torrent_files.extend( [ fullpath - for fullpathlist in self.pool.starmap(get_full_path_of_torrent_files, torrent_files_and_save_path) + for fullpathlist in self.executor.map(get_full_path_of_torrent_files, torrent_files_and_save_path) for fullpath in fullpathlist if fullpath not in torrent_files ] @@ -91,24 +91,22 @@ def rem_orphaned(self): self.config.send_notifications(attr) # Delete empty directories after moving orphan files if not self.config.dry_run: - orphaned_parent_path = set(self.pool.map(move_orphan, orphaned_files)) + orphaned_parent_path = set(self.executor.map(self.move_orphan, orphaned_files)) logger.print_line("Removing newly empty directories", self.config.loglevel) - self.pool.starmap(util.remove_empty_directories, zip(orphaned_parent_path, repeat("**/*"))) + self.executor.map(util.remove_empty_directories, zip(orphaned_parent_path, repeat("**/*"))) else: logger.print_line("No Orphaned Files found.", self.config.loglevel) - def cleanup_pool(self): - self.pool.close() - self.pool.join() + def move_orphan(self, file): + src = file.replace(self.root_dir, self.remote_dir) + dest = os.path.join(self.orphaned_dir, file.replace(self.root_dir, "")) + util.move_files(src, dest, True) + return os.path.dirname(file).replace(self.root_dir, self.remote_dir) -def init_pool(conf): - global _config - _config = conf - - -def get_full_path_of_torrent_files(torrent_files, save_path): +def get_full_path_of_torrent_files(torrent_files_and_save_path): + torrent_files, save_path = torrent_files_and_save_path fullpath_torrent_files = [] for file in torrent_files: fullpath = os.path.join(save_path, file) @@ -116,10 +114,3 @@ def get_full_path_of_torrent_files(torrent_files, save_path): fullpath = fullpath.replace(r"/", "\\") if ":\\" in fullpath else fullpath fullpath_torrent_files.append(fullpath) return fullpath_torrent_files - - -def move_orphan(file): - src = file.replace(_config.root_dir, _config.remote_dir) # Could be optimized to only run when root != remote - dest = os.path.join(_config.orphaned_dir, file.replace(_config.root_dir, "")) - util.move_files(src, dest, True) - return os.path.dirname(file).replace(_config.root_dir, _config.remote_dir) # Another candidate for micro optimizing From aee6a32e52704ed02db60fa309c0404345d0c3b9 Mon Sep 17 00:00:00 2001 From: bobokun Date: Mon, 22 May 2023 11:11:45 -0400 Subject: [PATCH 25/38] updates changelog for 3.6.3 --- CHANGELOG | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index d09f187f..626c9502 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,6 +1,14 @@ +# Requirements Updated +- pre-commit updated to 3.3.3 +- requests updated to 2.30.0 +- ruamel.yaml updated to 0.17.26 + # Bug Fixes -- Fixes bug in cross_seed (Fixes #270) -- Bug causing RecycleBin not to be created when full path is defined. (Fixes #271) -- Fixes Uncaught exception while emptying recycle bin (Fixes #272) +- Changes HardLink Logic (Thanks to @ColinHebert for the suggestion) Fixes #291 +- Additional error checking (Fixes #282) +- Fixes #287 (Thanks to @buthed010203 #290) +- Fixes Remove Orphan crashing when multiprocessing (Thanks to @buthed010203 #289) +- Fixes Remove Orphan from crashing in Windows (Fixes #275) +- Fixes #292 -**Full Changelog**: https://github.com/StuffAnThings/qbit_manage/compare/v3.6.1...v3.6.2 +**Full Changelog**: https://github.com/StuffAnThings/qbit_manage/compare/v3.6.2...v3.6.3 From 9684bf3e38215be1e3a6adb824763fb946085e34 Mon Sep 17 00:00:00 2001 From: bobokun Date: Mon, 22 May 2023 15:51:19 -0400 Subject: [PATCH 26/38] Fixes #201, Fixes #279 --- CHANGELOG | 3 +++ modules/core/cross_seed.py | 19 ++++++++++++++++++- modules/torrent_hash_generator.py | 25 +++++++++++++++++++++++++ qbit_manage.py | 12 ++++++------ requirements.txt | 1 + 5 files changed, 53 insertions(+), 7 deletions(-) create mode 100644 modules/torrent_hash_generator.py diff --git a/CHANGELOG b/CHANGELOG index 626c9502..58bc754f 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -2,6 +2,7 @@ - pre-commit updated to 3.3.3 - requests updated to 2.30.0 - ruamel.yaml updated to 0.17.26 +- Adds new dependency bencodepy to generate hash for cross-seed # Bug Fixes - Changes HardLink Logic (Thanks to @ColinHebert for the suggestion) Fixes #291 @@ -10,5 +11,7 @@ - Fixes Remove Orphan crashing when multiprocessing (Thanks to @buthed010203 #289) - Fixes Remove Orphan from crashing in Windows (Fixes #275) - Fixes #292 +- Fixes #201 +- Fixes #279 **Full Changelog**: https://github.com/StuffAnThings/qbit_manage/compare/v3.6.2...v3.6.3 diff --git a/modules/core/cross_seed.py b/modules/core/cross_seed.py index cbad7faa..ffbd6152 100644 --- a/modules/core/cross_seed.py +++ b/modules/core/cross_seed.py @@ -2,6 +2,7 @@ from collections import Counter from modules import util +from modules.torrent_hash_generator import TorrentHashGenerator logger = util.logger @@ -67,7 +68,23 @@ def cross_seed(self): self.client.torrents.add( torrent_files=src, save_path=dest, category=category, tags="cross-seed", is_paused=True ) - util.move_files(src, dir_cs_out) + self.qbt.torrentinfo[t_name]["count"] += 1 + try: + torrent_hash_generator = TorrentHashGenerator(src) + torrent_hash = torrent_hash_generator.generate_torrent_hash() + util.move_files(src, dir_cs_out) + except Exception as e: + logger.warning(f"Unable to generate torrent hash from cross-seed {t_name}: {e}") + try: + if torrent_hash: + torrent_info = self.qbt.get_torrents({"torrent_hashes": torrent_hash}) + except Exception as e: + logger.warning(f"Unable to find hash {torrent_hash} in qbt: {e}") + if torrent_info: + torrent = torrent_info[0] + self.qbt.torrentvalid.append(torrent) + self.qbt.torrentinfo[t_name]["torrents"].append(torrent) + self.qbt.torrent_list.append(torrent) else: logger.print_line(f"Found {t_name} in {dir_cs} but original torrent is not complete.", self.config.loglevel) logger.print_line("Not adding to qBittorrent", self.config.loglevel) diff --git a/modules/torrent_hash_generator.py b/modules/torrent_hash_generator.py new file mode 100644 index 00000000..d30bda8d --- /dev/null +++ b/modules/torrent_hash_generator.py @@ -0,0 +1,25 @@ +import hashlib + +import bencodepy + + +class TorrentHashGenerator: + def __init__(self, torrent_file_path): + self.torrent_file_path = torrent_file_path + + def generate_torrent_hash(self): + try: + with open(self.torrent_file_path, "rb") as torrent_file: + torrent_data = torrent_file.read() + + try: + torrent_info = bencodepy.decode(torrent_data) + info_data = bencodepy.encode(torrent_info[b"info"]) + info_hash = hashlib.sha1(info_data).hexdigest() + return info_hash + except KeyError: + raise ValueError("Invalid .torrent file format. 'info' key not found.") + except FileNotFoundError: + raise FileNotFoundError(f"Torrent file '{self.torrent_file_path}' not found.") + except Exception as e: + raise Exception(f"Error: {e}") diff --git a/qbit_manage.py b/qbit_manage.py index 0250a757..b1bd71d6 100755 --- a/qbit_manage.py +++ b/qbit_manage.py @@ -393,6 +393,12 @@ def finished_run(): if cfg.commands["tag_update"]: stats["tagged"] += Tags(qbit_manager).stats + # Set Cross Seed + if cfg.commands["cross_seed"]: + cross_seed = CrossSeed(qbit_manager) + stats["added"] += cross_seed.stats_added + stats["tagged"] += cross_seed.stats_tagged + # Remove Unregistered Torrents and tag errors if cfg.commands["rem_unregistered"] or cfg.commands["tag_tracker_error"]: rem_unreg = RemoveUnregistered(qbit_manager) @@ -403,12 +409,6 @@ def finished_run(): stats["untagged_tracker_error"] += rem_unreg.stats_untagged stats["tagged"] += rem_unreg.stats_tagged - # Set Cross Seed - if cfg.commands["cross_seed"]: - cross_seed = CrossSeed(qbit_manager) - stats["added"] += cross_seed.stats_added - stats["tagged"] += cross_seed.stats_tagged - # Recheck Torrents if cfg.commands["recheck"]: recheck = ReCheck(qbit_manager) diff --git a/requirements.txt b/requirements.txt index 3df979a8..a5f2395f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +bencodepy==0.9.5 flake8==6.0.0 pre-commit==3.3.2 qbittorrent-api==2023.4.47 From 99a7984693c931c42d1041ad488156a444e11e3c Mon Sep 17 00:00:00 2001 From: bobokun Date: Mon, 22 May 2023 16:16:38 -0400 Subject: [PATCH 27/38] use logging in torrent_hash_generator --- modules/torrent_hash_generator.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/modules/torrent_hash_generator.py b/modules/torrent_hash_generator.py index d30bda8d..69302614 100644 --- a/modules/torrent_hash_generator.py +++ b/modules/torrent_hash_generator.py @@ -2,6 +2,11 @@ import bencodepy +from modules import util +from modules.util import Failed + +logger = util.logger + class TorrentHashGenerator: def __init__(self, torrent_file_path): @@ -11,15 +16,15 @@ def generate_torrent_hash(self): try: with open(self.torrent_file_path, "rb") as torrent_file: torrent_data = torrent_file.read() - try: torrent_info = bencodepy.decode(torrent_data) info_data = bencodepy.encode(torrent_info[b"info"]) info_hash = hashlib.sha1(info_data).hexdigest() + logger.trace(f"info_hash: {info_hash}") return info_hash except KeyError: - raise ValueError("Invalid .torrent file format. 'info' key not found.") + logger.error("Invalid .torrent file format. 'info' key not found.") except FileNotFoundError: - raise FileNotFoundError(f"Torrent file '{self.torrent_file_path}' not found.") - except Exception as e: - raise Exception(f"Error: {e}") + logger.error(f"Torrent file '{self.torrent_file_path}' not found.") + except Failed as err: + logger.error(f"TorrentHashGenerator Error: {err}") From 8cd69a14c7fb3b132d26d686e4ca53bd34592946 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 23 May 2023 00:59:41 +0000 Subject: [PATCH 28/38] Bump dependabot/fetch-metadata from 1.4.0 to 1.5.0 Bumps [dependabot/fetch-metadata](https://github.com/dependabot/fetch-metadata) from 1.4.0 to 1.5.0. - [Release notes](https://github.com/dependabot/fetch-metadata/releases) - [Commits](https://github.com/dependabot/fetch-metadata/compare/v1.4.0...v1.5.0) --- updated-dependencies: - dependency-name: dependabot/fetch-metadata dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/dependabot-approve-and-auto-merge.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dependabot-approve-and-auto-merge.yml b/.github/workflows/dependabot-approve-and-auto-merge.yml index 04d0433e..9c059b76 100644 --- a/.github/workflows/dependabot-approve-and-auto-merge.yml +++ b/.github/workflows/dependabot-approve-and-auto-merge.yml @@ -19,7 +19,7 @@ jobs: # will not occur. - name: Dependabot metadata id: dependabot-metadata - uses: dependabot/fetch-metadata@v1.4.0 + uses: dependabot/fetch-metadata@v1.5.0 with: github-token: "${{ secrets.GITHUB_TOKEN }}" # Here the PR gets approved. From 8e10bb4a0791f5614a99a85d0d40093619df2ea3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 23 May 2023 00:59:57 +0000 Subject: [PATCH 29/38] Bump requests from 2.30.0 to 2.31.0 Bumps [requests](https://github.com/psf/requests) from 2.30.0 to 2.31.0. - [Release notes](https://github.com/psf/requests/releases) - [Changelog](https://github.com/psf/requests/blob/main/HISTORY.md) - [Commits](https://github.com/psf/requests/compare/v2.30.0...v2.31.0) --- updated-dependencies: - dependency-name: requests dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a5f2395f..a0a29bc8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ bencodepy==0.9.5 flake8==6.0.0 pre-commit==3.3.2 qbittorrent-api==2023.4.47 -requests==2.30.0 +requests==2.31.0 retrying==1.3.4 ruamel.yaml==0.17.26 schedule==1.2.0 From 27a04c5634bd43f39d746808a0ba0f0a2f042b0e Mon Sep 17 00:00:00 2001 From: bobokun Date: Mon, 22 May 2023 21:29:31 -0400 Subject: [PATCH 30/38] More info on versions --- .github/workflows/develop.yml | 2 ++ .github/workflows/version.yml | 2 ++ Dockerfile | 26 +++++++++++---- VERSION | 2 +- modules/__init__.py | 4 ++- modules/util.py | 59 +++++++++++++++++++++++++++++++++++ qbit_manage.py | 48 +++++++++++++++++++++++----- requirements.txt | 1 + 8 files changed, 129 insertions(+), 15 deletions(-) diff --git a/.github/workflows/develop.yml b/.github/workflows/develop.yml index 6ed64924..a60ab8bd 100644 --- a/.github/workflows/develop.yml +++ b/.github/workflows/develop.yml @@ -46,6 +46,8 @@ jobs: with: context: ./ file: ./Dockerfile + build-args: | + "BRANCH_NAME=develop" platforms: linux/amd64,linux/arm64,linux/arm/v7 push: true tags: ${{ secrets.DOCKER_HUB_USERNAME }}/qbit_manage:develop diff --git a/.github/workflows/version.yml b/.github/workflows/version.yml index 7b6b4b1e..39d7c823 100644 --- a/.github/workflows/version.yml +++ b/.github/workflows/version.yml @@ -14,6 +14,8 @@ jobs: - name: Check Out Repo uses: actions/checkout@v3 + with: + fetch-depth: 0 - name: Login to Docker Hub uses: docker/login-action@v2 diff --git a/Dockerfile b/Dockerfile index 1cabb91c..ef496ec0 100755 --- a/Dockerfile +++ b/Dockerfile @@ -1,13 +1,27 @@ -FROM python:3.10-alpine - -# install packages -RUN apk add --no-cache gcc g++ libxml2-dev libxslt-dev shadow bash curl wget jq grep sed coreutils findutils unzip p7zip ca-certificates +FROM python:3.11-slim-buster +ARG BRANCH_NAME=master +ENV BRANCH_NAME ${BRANCH_NAME} +ENV TINI_VERSION v0.19.0 +ENV QBM_DOCKER True COPY requirements.txt / -RUN echo "**** install python packages ****" \ +# install packages +RUN echo "**** install system packages ****" \ + && apt-get update \ + && apt-get upgrade -y --no-install-recommends \ + && apt-get install -y tzdata --no-install-recommends \ + && apt-get install -y --no-cache gcc g++ libxml2-dev libxslt-dev shadow bash curl wget jq grep sed coreutils findutils unzip p7zip ca-certificates \ + && wget -O /tini https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-"$(dpkg --print-architecture | awk -F- '{ print $NF }')" \ + && chmod +x /tini \ && pip3 install --no-cache-dir --upgrade --requirement /requirements.txt \ - && rm -rf /requirements.txt /tmp/* /var/tmp/* + && apt-get --purge autoremove gcc g++ libxml2-dev libxslt-dev libz-dev -y \ + && apt-get clean \ + && apt-get update \ + && apt-get check \ + && apt-get -f install \ + && apt-get autoclean \ + && rm -rf /requirements.txt /tmp/* /var/tmp/* /var/lib/apt/lists/* COPY . /app WORKDIR /app diff --git a/VERSION b/VERSION index 4a788a01..eacb9d84 100755 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.6.3 +3.6.3-develop1 diff --git a/modules/__init__.py b/modules/__init__.py index 78f14cd1..c5ce4e3c 100644 --- a/modules/__init__.py +++ b/modules/__init__.py @@ -13,8 +13,10 @@ with open(version_file_path) as f: version_str = f.read().strip() +# Get only the first 3 digits +version_str_split = version_str.rsplit("-", 1)[0] # Convert the version string to a tuple of integers -__version_info__ = tuple(map(int, version_str.split("."))) +__version_info__ = tuple(map(int, version_str_split.split("."))) # Define the version string using the version_info tuple __version__ = ".".join(str(i) for i in __version_info__) diff --git a/modules/util.py b/modules/util.py index c4734b4e..c4c5304c 100755 --- a/modules/util.py +++ b/modules/util.py @@ -7,6 +7,7 @@ import time from pathlib import Path +import requests import ruamel.yaml logger = logging.getLogger("qBit Manage") @@ -71,6 +72,64 @@ class TorrentMessages: ] +def guess_branch(version, env_version, git_branch): + if git_branch: + return git_branch + elif env_version == "develop": + return env_version + elif version[2] > 0: + dev_version = get_develop() + if version[1] != dev_version[1] or version[2] <= dev_version[2]: + return "develop" + else: + return "master" + + +def current_version(version, branch=None): + if branch == "develop": + return get_develop() + elif version[2] > 0: + new_version = get_develop() + if version[1] != new_version[1] or new_version[2] >= version[2]: + return new_version + else: + return get_master() + + +develop_version = None + + +def get_develop(): + global develop_version + if develop_version is None: + develop_version = get_version("develop") + return develop_version + + +master_version = None + + +def get_master(): + global master_version + if master_version is None: + master_version = get_version("master") + return master_version + + +def get_version(level): + try: + url = f"https://raw.githubusercontent.com/StuffAnThings/qbit_manage/{level}/VERSION" + return parse_version(requests.get(url).content.decode().strip(), text=level) + except requests.exceptions.ConnectionError: + return "Unknown", "Unknown", 0 + + +def parse_version(version, text="develop"): + version = version.replace("develop", text) + split_version = version.split(f"-{text}") + return version, split_version[0], int(split_version[1]) if len(split_version) > 1 else 0 + + class check: """Check for attributes in config.""" diff --git a/qbit_manage.py b/qbit_manage.py index b1bd71d6..32106c91 100755 --- a/qbit_manage.py +++ b/qbit_manage.py @@ -3,6 +3,7 @@ import argparse import glob import os +import platform import sys import time from datetime import datetime @@ -166,16 +167,23 @@ args = parser.parse_args() +static_envs = [] +test_value = None + + def get_arg(env_str, default, arg_bool=False, arg_int=False): - """Get argument from environment variable or command line argument.""" + global test_value env_vars = [env_str] if not isinstance(env_str, list) else env_str final_value = None + static_envs.extend(env_vars) for env_var in env_vars: env_value = os.environ.get(env_var) + if env_var == "BRANCH_NAME": + test_value = env_value if env_value is not None: final_value = env_value break - if final_value is not None: + if final_value or (arg_int and final_value == 0): if arg_bool: if final_value is True or final_value is False: return final_value @@ -184,13 +192,28 @@ def get_arg(env_str, default, arg_bool=False, arg_int=False): else: return False elif arg_int: - return int(final_value) + try: + return int(final_value) + except ValueError: + return default else: return str(final_value) else: return default +try: + from git import Repo, InvalidGitRepositoryError + + try: + git_branch = Repo(path=".").head.ref.name # noqa + except InvalidGitRepositoryError: + git_branch = None +except ImportError: + git_branch = None + +env_version = get_arg("BRANCH_NAME", "master") +is_docker = get_arg("QBM_DOCKER", False, arg_bool=True) run = get_arg("QBT_RUN", args.run, arg_bool=True) sch = get_arg("QBT_SCHEDULE", args.min) startupDelay = get_arg("QBT_STARTUP_DELAY", args.startupDelay) @@ -306,13 +329,15 @@ def my_except_hook(exctype, value, tbi): sys.excepthook = my_except_hook -version = "Unknown" +version = ("Unknown", "Unknown", 0) with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), "VERSION")) as handle: for line in handle.readlines(): line = line.strip() if len(line) > 0: - version = line + version = util.parse_version(line) break +branch = util.guess_branch(version, env_version, git_branch) +version = (version[0].replace("develop", branch), version[1].replace("develop", branch), version[2]) def start_loop(): @@ -521,8 +546,17 @@ def calc_next_run(schd, write_out=False): logger.info_center(r" \__, |_.__/|_|\__| |_| |_| |_|\__,_|_| |_|\__,_|\__, |\___|") # noqa: W605 logger.info_center(" | | ______ __/ | ") # noqa: W605 logger.info_center(" |_| |______| |___/ ") # noqa: W605 - logger.info(f" Version: {version}") - + system_ver = "Docker" if is_docker else f"Python {platform.python_version()}" + logger.info(f" Version: {version[0]} ({system_ver}){f' (Git: {git_branch})' if git_branch else ''}") + latest_version = util.current_version(version, branch=branch) + new_version = ( + latest_version[0] + if latest_version and (version[1] != latest_version[1] or (version[2] and version[2] < latest_version[2])) + else None + ) + if new_version: + logger.info(f" Newest Version: {new_version}") + logger.info(f" Platform: {platform.platform()}") logger.separator(loglevel="DEBUG") logger.debug(f" --run (QBT_RUN): {run}") logger.debug(f" --schedule (QBT_SCHEDULE): {sch}") diff --git a/requirements.txt b/requirements.txt index a0a29bc8..17d25111 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ bencodepy==0.9.5 flake8==6.0.0 +GitPython==3.1.31 pre-commit==3.3.2 qbittorrent-api==2023.4.47 requests==2.31.0 From f85adbbb02e8c5173e980c88dcbe68a3dcd97819 Mon Sep 17 00:00:00 2001 From: bobokun Date: Mon, 22 May 2023 21:33:42 -0400 Subject: [PATCH 31/38] fix dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index ef496ec0..60482e9b 100755 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,7 @@ RUN echo "**** install system packages ****" \ && apt-get update \ && apt-get upgrade -y --no-install-recommends \ && apt-get install -y tzdata --no-install-recommends \ - && apt-get install -y --no-cache gcc g++ libxml2-dev libxslt-dev shadow bash curl wget jq grep sed coreutils findutils unzip p7zip ca-certificates \ + && apt-get install -y gcc g++ libxml2-dev libxslt-dev bash curl wget jq grep sed coreutils findutils unzip p7zip ca-certificates \ && wget -O /tini https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-"$(dpkg --print-architecture | awk -F- '{ print $NF }')" \ && chmod +x /tini \ && pip3 install --no-cache-dir --upgrade --requirement /requirements.txt \ From 4c7c54f1cb4a127c388f2f8c65004e47b7cf15ab Mon Sep 17 00:00:00 2001 From: bobokun Date: Mon, 22 May 2023 21:37:56 -0400 Subject: [PATCH 32/38] updates dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 60482e9b..e35cb1af 100755 --- a/Dockerfile +++ b/Dockerfile @@ -26,4 +26,4 @@ RUN echo "**** install system packages ****" \ COPY . /app WORKDIR /app VOLUME /config -ENTRYPOINT ["python3", "qbit_manage.py"] +ENTRYPOINT ["/tini", "-s", "python3", "qbit_manage.py", "--"] From b7af4de4ac75d2e1ada1e6d307c32917a808becd Mon Sep 17 00:00:00 2001 From: bobokun Date: Mon, 22 May 2023 21:50:29 -0400 Subject: [PATCH 33/38] changelog and version bump --- CHANGELOG | 2 ++ Dockerfile | 2 +- VERSION | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 58bc754f..a40e2d93 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -3,6 +3,7 @@ - requests updated to 2.30.0 - ruamel.yaml updated to 0.17.26 - Adds new dependency bencodepy to generate hash for cross-seed +- Adds new dependency GitPython for checking git branches # Bug Fixes - Changes HardLink Logic (Thanks to @ColinHebert for the suggestion) Fixes #291 @@ -13,5 +14,6 @@ - Fixes #292 - Fixes #201 - Fixes #279 +- Updates Dockerfile to debloat and move to Python 3.11 **Full Changelog**: https://github.com/StuffAnThings/qbit_manage/compare/v3.6.2...v3.6.3 diff --git a/Dockerfile b/Dockerfile index e35cb1af..1c6afc94 100755 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,7 @@ RUN echo "**** install system packages ****" \ && apt-get update \ && apt-get upgrade -y --no-install-recommends \ && apt-get install -y tzdata --no-install-recommends \ - && apt-get install -y gcc g++ libxml2-dev libxslt-dev bash curl wget jq grep sed coreutils findutils unzip p7zip ca-certificates \ + && apt-get install -y gcc g++ libxml2-dev libxslt-dev libz-dev bash curl wget jq grep sed coreutils findutils unzip p7zip ca-certificates \ && wget -O /tini https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-"$(dpkg --print-architecture | awk -F- '{ print $NF }')" \ && chmod +x /tini \ && pip3 install --no-cache-dir --upgrade --requirement /requirements.txt \ diff --git a/VERSION b/VERSION index eacb9d84..bb6360fe 100755 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.6.3-develop1 +3.6.3-develop2 From afd2e64dd9df71e89a433a134115f1eaa51eaf62 Mon Sep 17 00:00:00 2001 From: buthed010203 Date: Tue, 23 May 2023 09:15:11 -0400 Subject: [PATCH 34/38] Fix update_orphaned empty directory removal As this no longer uses a starmap, the second argument wasn't being passed in correctly. --- modules/core/remove_orphaned.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/modules/core/remove_orphaned.py b/modules/core/remove_orphaned.py index 772ccc25..a49ddc6d 100644 --- a/modules/core/remove_orphaned.py +++ b/modules/core/remove_orphaned.py @@ -93,8 +93,7 @@ def rem_orphaned(self): if not self.config.dry_run: orphaned_parent_path = set(self.executor.map(self.move_orphan, orphaned_files)) logger.print_line("Removing newly empty directories", self.config.loglevel) - self.executor.map(util.remove_empty_directories, zip(orphaned_parent_path, repeat("**/*"))) - + self.executor.map(lambda dir: util.remove_empty_directories(dir, "**/*"), orphaned_parent_path) else: logger.print_line("No Orphaned Files found.", self.config.loglevel) From 02baffd29b3c445b5507eec04b21a30819d0b545 Mon Sep 17 00:00:00 2001 From: buthed010203 Date: Tue, 23 May 2023 09:18:18 -0400 Subject: [PATCH 35/38] Cleanup imports --- modules/core/remove_orphaned.py | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/core/remove_orphaned.py b/modules/core/remove_orphaned.py index a49ddc6d..88fcd96f 100644 --- a/modules/core/remove_orphaned.py +++ b/modules/core/remove_orphaned.py @@ -1,7 +1,6 @@ import os from concurrent.futures import ThreadPoolExecutor from fnmatch import fnmatch -from itertools import repeat from modules import util From 2fd5701922cf305fabc40f97e2f6c7bd84b95bae Mon Sep 17 00:00:00 2001 From: buthed010203 Date: Tue, 23 May 2023 10:12:51 -0400 Subject: [PATCH 36/38] Make remove_orphaned even faster This is not an addiction I swear --- modules/core/remove_orphaned.py | 37 ++++++++++++++------------------- 1 file changed, 16 insertions(+), 21 deletions(-) diff --git a/modules/core/remove_orphaned.py b/modules/core/remove_orphaned.py index 88fcd96f..93dfb4de 100644 --- a/modules/core/remove_orphaned.py +++ b/modules/core/remove_orphaned.py @@ -28,31 +28,24 @@ def rem_orphaned(self): self.stats = 0 logger.separator("Checking for Orphaned Files", space=False, border=False) torrent_files = [] - root_files = [] orphaned_files = [] excluded_orphan_files = [] - root_files = util.get_root_files(self.remote_dir, self.root_dir, self.orphaned_dir) + root_files = self.executor.submit(util.get_root_files, self.remote_dir, self.root_dir, self.orphaned_dir) # Get an updated list of torrents logger.print_line("Locating orphan files", self.config.loglevel) torrent_list = self.qbt.get_torrents({"sort": "added_on"}) - torrent_files_and_save_path = [] - for torrent in torrent_list: - torrent_files = [] - for torrent_files_dict in torrent.files: - torrent_files.append(torrent_files_dict.name) - torrent_files_and_save_path.append((torrent_files, torrent.save_path)) + torrent_files.extend( [ fullpath - for fullpathlist in self.executor.map(get_full_path_of_torrent_files, torrent_files_and_save_path) + for fullpathlist in self.executor.map(self.get_full_path_of_torrent_files, torrent_list) for fullpath in fullpathlist - if fullpath not in torrent_files ] ) - orphaned_files = set(root_files) - set(torrent_files) + orphaned_files = set(root_files.result()) - set(torrent_files) if self.config.orphaned["exclude_patterns"]: logger.print_line("Processing orphan exclude patterns") @@ -93,6 +86,7 @@ def rem_orphaned(self): orphaned_parent_path = set(self.executor.map(self.move_orphan, orphaned_files)) logger.print_line("Removing newly empty directories", self.config.loglevel) self.executor.map(lambda dir: util.remove_empty_directories(dir, "**/*"), orphaned_parent_path) + else: logger.print_line("No Orphaned Files found.", self.config.loglevel) @@ -102,13 +96,14 @@ def move_orphan(self, file): util.move_files(src, dest, True) return os.path.dirname(file).replace(self.root_dir, self.remote_dir) - -def get_full_path_of_torrent_files(torrent_files_and_save_path): - torrent_files, save_path = torrent_files_and_save_path - fullpath_torrent_files = [] - for file in torrent_files: - fullpath = os.path.join(save_path, file) - # Replace fullpath with \\ if qbm is running in docker (linux) but qbt is on windows - fullpath = fullpath.replace(r"/", "\\") if ":\\" in fullpath else fullpath - fullpath_torrent_files.append(fullpath) - return fullpath_torrent_files + def get_full_path_of_torrent_files(self, torrent): + torrent_files = map(lambda dict: dict.name, torrent.files) + save_path = torrent.save_path + + fullpath_torrent_files = [] + for file in torrent_files: + fullpath = os.path.join(save_path, file) + # Replace fullpath with \\ if qbm is running in docker (linux) but qbt is on windows + fullpath = fullpath.replace(r"/", "\\") if ":\\" in fullpath else fullpath + fullpath_torrent_files.append(fullpath) + return fullpath_torrent_files From 19fdd795ae5d90f737c618cfa80fd0e7511ee25b Mon Sep 17 00:00:00 2001 From: buthed010203 Date: Tue, 23 May 2023 10:26:01 -0400 Subject: [PATCH 37/38] Update VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index bb6360fe..0572169b 100755 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.6.3-develop2 +3.6.3-develop3 From aaf2a02b8310f466fe6ce8e84b7f3da78976c9b8 Mon Sep 17 00:00:00 2001 From: bobokun Date: Tue, 23 May 2023 20:47:38 -0400 Subject: [PATCH 38/38] 3.6.3 --- CHANGELOG | 3 ++- VERSION | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index a40e2d93..197cd8b7 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,6 +1,6 @@ # Requirements Updated - pre-commit updated to 3.3.3 -- requests updated to 2.30.0 +- requests updated to 2.31.0 - ruamel.yaml updated to 0.17.26 - Adds new dependency bencodepy to generate hash for cross-seed - Adds new dependency GitPython for checking git branches @@ -10,6 +10,7 @@ - Additional error checking (Fixes #282) - Fixes #287 (Thanks to @buthed010203 #290) - Fixes Remove Orphan crashing when multiprocessing (Thanks to @buthed010203 #289) +- Speed optimization for Remove Orphan (Thanks to @buthed010203 #299) - Fixes Remove Orphan from crashing in Windows (Fixes #275) - Fixes #292 - Fixes #201 diff --git a/VERSION b/VERSION index 0572169b..4a788a01 100755 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.6.3-develop3 +3.6.3