Merge branch 'test'

.dockerignore

@@ -7,3 +7,6 @@
 !requirements.txt
 !poetry.lock
 !pyproject.toml
+!package.json
+!package-lock.json
+!tests/screenshot.js

.github/ci-upgrade.yaml

@@ -0,0 +1,3 @@
+values:
+  matchStabilizationBranches:
+    - master

.github/workflows/clean.yaml

@@ -13,7 +13,7 @@ jobs:
     timeout-minutes: 5
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - run: docker login --username=${{ secrets.DOCKER_USERNAME }} --password=${{ secrets.DOCKER_PASSWORD }}
 

.github/workflows/codeql.yaml

@@ -21,7 +21,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         if: env.HAS_SECRETS == 'HAS_SECRETS'
 
       - name: Initialize CodeQL

.github/workflows/main.yaml

@@ -21,10 +21,9 @@ jobs:
 
     steps:
       - run: docker system prune --all --force
-
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         if: env.HAS_SECRETS != 'HAS_SECRETS'
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           token: ${{ secrets.TOKEN }}
           fetch-depth: 0
@@ -39,28 +38,27 @@ jobs:
           key: pre-commit-${{ hashFiles('.pre-commit-config.yaml') }}
           restore-keys: "pre-commit-${{ hashFiles('.pre-commit-config.yaml') }}\npre-commit-"
       - run: pre-commit run --all-files
-      - name: Checks
-        run: c2cciutils-checks
         env:
-          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
+          SKIP: poetry-lock
+      - run: git diff && false
+        if: failure()
+
+      - name: Print environment information
+        run: c2cciutils-env
 
       - name: GitHub event
         run: echo ${GITHUB_EVENT} | python3 -m json.tool
         env:
           GITHUB_EVENT: ${{ toJson(github) }}
 
-          # the if the generated files are up to date
-      - run: make jsonschema
-      - run: git diff --quiet
-      - run: git diff
-        if: failure()
-
       - run: poetry-dynamic-versioning
 
       - run: make build
       - run: make build-tests
       - run: make prospector
-      - name: pytest
+      - run: npm install
+      - run: mkdir -p results/status
+      - name: Pytest
         run: >
           docker run --rm --env=PYTHONPATH=/opt/
           --volume=$(pwd)/results:/results --volume=$(pwd)/tests:/tests
@@ -70,6 +68,9 @@ jobs:
           --profile --profile-svg --junitxml=/results/test-reports/junit.xml .'
       - run: sudo chmod -R go+r results || true
         if: failure()
+      - name: Get the status images
+        run: cp tests/status/*.png results/status/
+        if: failure()
       - uses: actions/upload-artifact@v3
         with:
           name: results
@@ -81,11 +82,10 @@ jobs:
       - run: docker login --username=${{ secrets.DOCKER_USERNAME }} --password=${{ secrets.DOCKER_PASSWORD }}
 
       - name: Init pypi
-        run: |
+        run: |-
           echo "[pypi]" > ~/.pypirc
           echo "username = ${{ secrets.PYPI_USERNAME }}" >> ~/.pypirc
           echo "password = ${{ secrets.PYPI_PASSWORD }}" >> ~/.pypirc
-          pip install --user wheel twine
         if: env.HAS_SECRETS == 'HAS_SECRETS'
 
       - name: Init Git
@@ -99,7 +99,7 @@ jobs:
         if: env.HAS_SECRETS == 'HAS_SECRETS'
         env:
           SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
-          GITHUB_TOKEN: ${{ secrets.TOKEN_NEW }}
+          GITHUB_TOKEN: ${{ secrets.TOKEN }}
 
       - run: make build-all
 
@@ -109,4 +109,4 @@ jobs:
         if: env.HAS_SECRETS == 'HAS_SECRETS'
         env:
           SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
-          GITHUB_TOKEN: ${{ secrets.TOKEN_NEW }}
+          GITHUB_TOKEN: ${{ secrets.TOKEN }}

.github/workflows/pr-checks.yaml

@@ -7,7 +7,7 @@ on:
       - reopened
       - labeled
       - unlabeled
-      - edit
+      - edited
       - synchronize
 
 jobs:
@@ -18,7 +18,7 @@ jobs:
 
     steps:
       - run: pip install --upgrade attrs
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - run: echo "${HOME}/.local/bin" >> ${GITHUB_PATH}
       - run: python3 -m pip install --user --pre c2cciutils[pr_checks]

.gitignore

@@ -1,8 +1,16 @@
 /results/
 /dist/
 __pycache__
+/.mypy_cache/
 /scan_to_paperless.egg-info/
 /tests/prof/
-/tests/img-0.png
-/tests/img-1.png
 /.github/changelog-generator-cache
+/docker-compose.override.yaml
+/node_modules/
+/tests/*.png
+!/tests/*.expected.png
+!/tests/*.mask.png
+/tests/status/*.png
+!/tests/status/*.expected.png
+!/tests/status/*.mask.png
+/tests/status/scan/status.html

.pre-commit-config.yaml

@@ -1,8 +1,58 @@
 ci:
+  autoupdate_schedule: quarterly
   skip:
+    - copyright
+    - poetry-check
+    - poetry-lock
     - ripsecrets
+    - schema-copy
+    - jsonschema-validator
+    - jsonschema-gentypes
+    - jsonschema2md
 
 repos:
+  - repo: local
+    hooks:
+      - id: schema-copy
+        name: Copy between schemas
+        entry: ci/schema-copy
+        language: python
+        files: ^scan_to_paperless/config_schema\.json$
+        pass_filenames: false
+
+  - repo: https://github.com/sbrunner/jsonschema-validator
+    rev: 0.1.0
+    hooks:
+      - id: jsonschema-validator
+        files: |-
+          (?x)^(
+              ci/config\.yaml
+          )$
+  - repo: https://github.com/sbrunner/jsonschema2md2
+    rev: 1.0.0
+    hooks:
+      - id: jsonschema2md
+        files: schema.json
+        args:
+          - --pre-commit
+          - scan_to_paperless/config_schema.json
+          - config.md
+      - id: jsonschema2md
+        files: schema.json
+        args:
+          - --pre-commit
+          - scan_to_paperless/process_schema.json
+          - process.md
+  - repo: https://github.com/camptocamp/jsonschema-gentypes
+    rev: 2.3.0 # Use the ref you want to point at
+    hooks:
+      - id: jsonschema-gentypes
+        files: |-
+          (?x)^(
+            jsonschema-gentypes\.yaml
+            |scan_to_paperless/process_schema\.json
+            |scan_to_paperless/config_schema\.json
+          )$
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.4.0
     hooks:
@@ -16,27 +66,34 @@ repos:
       - id: end-of-file-fixer
       - id: trailing-whitespace
       - id: mixed-line-ending
-  - repo: https://github.com/sbrunner/pre-commit-copyright
-    rev: 0.2.3
+  - repo: https://github.com/sbrunner/hooks
+    rev: 0.5.0
     hooks:
       - id: copyright
+      - id: workflows-require-timeout
+      - id: poetry-check
+        additional_dependencies:
+          - poetry==1.6.1 # pypi
+      - id: poetry-lock
+        additional_dependencies:
+          - poetry==1.6.1 # pypi
   - repo: https://github.com/codespell-project/codespell
     rev: v2.2.6
     hooks:
       - id: codespell
-        exclude: |
-          (?x)^(
-            (.*/)?poetry\.lock
-          )$
+        exclude: ^(.*/)?poetry\.lock$
+        args:
+          - --ignore-words=.github/spell-ignore-words.txt
   - repo: https://github.com/pre-commit/mirrors-prettier
-    rev: v2.7.1
+    rev: v3.0.3
     hooks:
       - id: prettier
         additional_dependencies:
-          - prettier@2.8.8 # npm
-          - prettier-plugin-sh@0.12.8 # npm
-          - prettier-plugin-toml@0.3.5 # npm
+          - prettier@3.0.3 # npm
+          - prettier-plugin-sh@0.13.1 # npm
+          - prettier-plugin-toml@1.0.0 # npm
           - '@prettier/plugin-xml@0.12.0' # npm
+        exclude: ^scan_to_paperless/status\.html$
   - repo: https://github.com/shellcheck-py/shellcheck-py
     rev: v0.9.0.6
     hooks:
@@ -46,41 +103,54 @@ repos:
     hooks:
       - id: git-check
   - repo: https://github.com/python-jsonschema/check-jsonschema
-    rev: 0.21.0
+    rev: 0.27.0
     hooks:
       - id: check-github-workflows
       - id: check-renovate
         additional_dependencies:
           - pyjson5==1.6.4 # pypi
+      - id: check-github-actions
+      - id: check-jsonschema
+        name: Check GitHub Workflows set timeout-minutes
+        files: ^\.github/workflows/[^/]+$
+        types:
+          - yaml
+        args:
+          - --builtin-schema
+          - github-workflows-require-timeout
   - repo: https://github.com/sirwart/ripsecrets
     rev: v0.1.7
     hooks:
       - id: ripsecrets
-        exclude: "(^\
-          thepaperlessproject-paperless-webserver.json\
-          |thepaperlessproject-paperless-consumer.json)"
+        exclude: |-
+          (?x)(
+            ^thepaperlessproject-paperless-webserver.json
+            |thepaperlessproject-paperless-consumer.json
+          )
+  - repo: https://github.com/asottile/pyupgrade
+    rev: v3.13.0
+    hooks:
+      - id: pyupgrade
+        args:
+          - --py39-plus
+  - repo: https://github.com/PyCQA/autoflake
+    rev: v2.2.1
+    hooks:
+      - id: autoflake
   - repo: https://github.com/psf/black
-    rev: 23.1.0
+    rev: 23.9.1
     hooks:
       - id: black
   - repo: https://github.com/PyCQA/isort
     rev: 5.12.0
     hooks:
       - id: isort
-  - repo: https://github.com/PyCQA/autoflake
-    rev: v2.0.2
-    hooks:
-      - id: autoflake
-  - repo: https://github.com/asottile/pyupgrade
-    rev: v3.3.2
-    hooks:
-      - id: pyupgrade
-        args:
-          - --py38-plus
   - repo: https://github.com/PyCQA/prospector
-    rev: v1.8.4
+    rev: 1.10.2
     hooks:
       - id: prospector
         args:
           - --tool=pydocstyle
           - --die-on-tool-error
+        additional_dependencies:
+          - prospector-profile-duplicated==0.3.0 # pypi

.prospector.yaml

@@ -1,4 +1,5 @@
----
+inherits:
+  - duplicated
 strictness: veryhigh
 doc-warnings: true
 max-line-length: 110
@@ -9,13 +10,13 @@ ignore-paths:
 
 pycodestyle:
   disable:
-    - E203 # whitespace before ':'
     - E501 # line too long, redundant with Black
 
 pydocstyle:
   disable:
     - D212 # Multi-line docstring summary should start at the first line
     - D203 # 1 blank line required before class docstring (found 0), not compatible with Black
+    - D202 # No blank lines allowed after function docstring
 
 pylint:
   disable:
@@ -27,7 +28,7 @@ pylint:
     - too-many-nested-blocks
     - too-many-arguments
     - too-many-return-statements
-    - wrong-import-order
+    - too-many-instance-attributes
     - line-too-long
     - no-value-for-parameter
     - no-else-continue