diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..d17b40d --- /dev/null +++ b/.coveragerc @@ -0,0 +1,9 @@ +[run] +branch = True +[report] +exclude_also = + if TYPE_CHECKING: + except ImportError: + raise NotImplementedError + raise AssertionError + \.\.\. diff --git a/.github/workflows/upload-tests.yml b/.github/workflows/upload-tests.yml new file mode 100644 index 0000000..2f3c7f3 --- /dev/null +++ b/.github/workflows/upload-tests.yml @@ -0,0 +1,54 @@ +name: grip-on-software/upload +on: + push: + branches: + - master + pull_request: + types: [opened, synchronize, reopened] +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: checkout + uses: actions/checkout@v4.1.4 + with: + fetch-depth: 0 + - uses: actions/setup-python@v5.1.0 + with: + python-version: "${{ matrix.python }}" + - name: Install development packages for some dependencies + run: | + sudo apt-get update + sudo apt-get install gnupg libgpg-error-dev libgpgme-dev libssl-dev swig + - name: Install dependencies for test + run: make setup_test + - name: Unit test and coverage + run: make coverage + env: + WEBTEST_INTERACTIVE: "False" + - name: Mypy typing coverage + run: | + make setup_analysis + make mypy + - name: Adjust source paths in coverage for Sonar + run: | + sed -i "s/\/home\/runner\/work\/upload\/upload/\/github\/workspace/g" \ + test-reports/cobertura.xml mypy-report/cobertura.xml + - name: SonarCloud Scan + uses: sonarsource/sonarcloud-github-action@v2.1.1 + if: "${{ matrix.python == '3.8.18' }}" + env: + SONAR_TOKEN: "${{ secrets.SONAR_TOKEN }}" + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + - name: Coveralls upload + run: | + pip install coveralls + coveralls + if: "${{ success() }}" + env: + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + strategy: + matrix: + python: + - '3.8.18' + - '3.12.3' diff --git a/.gitignore b/.gitignore index a7b1e24..a8dc12c 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,45 @@ env *.cfg !.isort.cfg +!setup.cfg *.log -env/ + +# Uploaded data data/ +upload/ + +# Pip installation +src/*/ +src/pip-delete-this-directory.txt + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +!typeshed/*/lib +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg + +# Typing coverage +.mypy_cache/ +mypy-report/ + +# Pylint +pylint-report.txt + +# Unit tests and coverage +.coverage +htmlcov/ +test-reports/ diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..ebe4878 --- /dev/null +++ b/Makefile @@ -0,0 +1,132 @@ +PACKAGE=gros-upload +MODULE=upload +COVERAGE=coverage +MYPY=mypy +PIP=python -m pip +PYLINT=pylint +RM=rm -rf +SOURCES_ANALYSIS=encrypted_upload test +SOURCES_COVERAGE=encrypted_upload,test +TEST=-m pytest -s test +TEST_OUTPUT=--junit-xml=test-reports/TEST-pytest.xml +TWINE=twine + +.PHONY: all +all: coverage mypy pylint + +.PHONY: release +release: test mypy pylint clean build tag push upload + +.PHONY: setup +setup: + $(PIP) install -r requirements.txt + +.PHONY: setup_release +setup_release: + $(PIP) install -r requirements-release.txt + +.PHONY: setup_analysis +setup_analysis: + $(PIP) install -r requirements-analysis.txt + +.PHONY: setup_test +setup_test: + $(PIP) install -r requirements-test.txt + +.PHONY: install +install: + $(PIP) install . + +.PHONY: pylint +pylint: + $(PYLINT) $(SOURCES_ANALYSIS) --exit-zero --reports=n \ + --msg-template="{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}" \ + -d duplicate-code + +.PHONY: mypy +mypy: + $(MYPY) $(SOURCES_ANALYSIS) \ + --cobertura-xml-report mypy-report \ + --junit-xml mypy-report/TEST-junit.xml \ + --no-incremental --show-traceback + +.PHONY: mypy_html +mypy_html: + $(MYPY) $(SOURCES_ANALYSIS) \ + --html-report mypy-report \ + --cobertura-xml-report mypy-report \ + --junit-xml mypy-report/TEST-junit.xml \ + --no-incremental --show-traceback + +.PHONY: test +test: + python $(TEST) $(TEST_OUTPUT) + +.PHONY: coverage +coverage: + $(COVERAGE) run --source=$(SOURCES_COVERAGE) $(TEST) $(TEST_OUTPUT) + $(COVERAGE) report -m + $(COVERAGE) xml -i -o test-reports/cobertura.xml + +# Version of the coverage target that does not write JUnit/cobertura XML output +.PHONY: cover +cover: + $(COVERAGE) run --source=$(SOURCES_COVERAGE) $(TEST) + $(COVERAGE) report -m + +.PHONY: get_version +get_version: get_toml_version get_init_version get_sonar_version get_citation_version + if [ "${TOML_VERSION}" != "${INIT_VERSION}" ] || [ "${TOML_VERSION}" != "${SONAR_VERSION}" ] || [ "${TOML_VERSION}" != "${CITATION_VERSION}" ]; then \ + echo "Version mismatch"; \ + exit 1; \ + fi + $(eval VERSION=$(TOML_VERSION)) + +.PHONY: get_init_version +get_init_version: + $(eval INIT_VERSION=v$(shell grep __version__ $(MODULE)/__init__.py | sed -E "s/__version__ = .([0-9.]+)./\\1/")) + $(info Version in __init__.py: $(INIT_VERSION)) + if [ -z "${INIT_VERSION}" ]; then \ + echo "Could not parse version"; \ + exit 1; \ + fi + +.PHONY: get_toml_version +get_toml_version: + $(eval TOML_VERSION=v$(shell grep "^version" pyproject.toml | sed -E "s/version = .([0-9.]+)./\\1/")) + $(info Version in pyproject.toml: $(TOML_VERSION)) + +.PHONY: get_sonar_version +get_sonar_version: + $(eval SONAR_VERSION=v$(shell grep projectVersion sonar-project.properties | cut -d= -f2)) + $(info Version in sonar-project.properties: $(SONAR_VERSION)) + +.PHONY: get_citation_version +get_citation_version: + $(eval CITATION_VERSION=v$(shell grep "^version:" CITATION.cff | cut -d' ' -f2)) + $(info Version in CITATION.cff: $(CITATION_VERSION)) + +.PHONY: tag +tag: get_version + git tag $(VERSION) + +.PHONY: build +build: + python -m build + +.PHONY: push +push: get_version + git push origin $(VERSION) + +.PHONY: upload +upload: + $(TWINE) upload dist/* + +.PHONY: clean +clean: + # Typing coverage and Pylint + $(RM) .mypy_cache mypy-report/ pylint-report.txt + # Tests + $(RM) test/sample/upload/ + # Pip and distribution + $(RM) src/ build/ dist/ $(PACKAGE).egg-info/ diff --git a/README.md b/README.md index 73473ee..e55b5f2 100644 --- a/README.md +++ b/README.md @@ -1,26 +1,70 @@ # Encrypted file upload server +[![PyPI](https://img.shields.io/pypi/v/gros-upload.svg)](https://pypi.python.org/pypi/gros-upload) +[![Build +status](https://github.com/grip-on-software/upload/actions/workflows/upload-tests.yml/badge.svg)](https://github.com/grip-on-software/upload/actions/workflows/upload-tests.yml) +[![Coverage +Status](https://coveralls.io/repos/github/grip-on-software/upload/badge.svg?branch=master)](https://coveralls.io/github/grip-on-software/upload?branch=master) +[![Quality Gate +Status](https://sonarcloud.io/api/project_badges/measure?project=grip-on-software_upload&metric=alert_status)](https://sonarcloud.io/project/overview?id=grip-on-software_upload) +[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.12784820.svg)](https://doi.org/10.5281/zenodo.12784820) + This repository includes a service for running a HTTP server which accepts -uploads of GPG-encrypted files. The service uses a keychain to keep GPG -passphrases. Certain uploaded files can be used to import a database dump. -Usual deployment setups would host this service behind a reverse proxy such as -NGINX or Apache which handles SSL termination and access control. +uploads of GPG-encrypted files. Although available as a package, it is mostly +meant to run as a standalone program or service. The application uses +a keychain to keep GPG passphrases which can be modified using a subcommand. +Certain uploaded files can be used to import a database dump. Usual deployment +setups would host this service behind a reverse proxy such as NGINX or Apache +which handles SSL termination and additional access control over the +Digest-based user authentication in this server. -## Requirements +## Installation -A working version of the [GPG -exchange](https://github.com/lhelwerd/gpg-exchange) library is required. Follow -the instructions there to install the GPG dependencies. +The [GPG exchange](https://github.com/lhelwerd/gpg-exchange) library is +required to be in a working state. Follow the instructions there to install the +GPG dependencies first. Then, to install the latest release version of the +packaged program from PyPI, run the following command: -Then install all Python dependencies using the following command: +``` +pip install gros-upload +``` -`pip install -r requirements.txt` +Another option is to build the program from this repository, which allows using +the most recent development code. Run `make setup` to install the dependencies. +The upload server itself may then be installed with `make install`, which +places the package in your current environment. We recommend using a virtual +environment during development. ## Configuration -Configure server settings in `upload.cfg` by copying `upload.cfg.example` and -replacing the variables with actual values. The following configuration -sections and items are known: +Configure server settings in `upload.cfg` by copying `upload.cfg.example` or +the example file below: +```ini +[server] +key = $SERVER_KEY +engine = $SERVER_ENGINE +files = $SERVER_FILES +secret = $SERVER_SECRET +keyring = $SERVER_KEYRING +realm = $SERVER_REALM + +[import] +database = $IMPORT_DATABASE +dump = $IMPORT_DUMP +path = $IMPORT_PATH +script = $IMPORT_SCRIPT + +[client] +$CLIENT_ID=$CLIENT_NAME + +[auth] +$CLIENT_ID=$CLIENT_AUTH + +[symm] +$CLIENT_ID=$CLIENT_PASSPHRASE +``` +Replace the variables with actual values. The following configuration sections +and items are known: - `server`: Configuration of the listener server. - `key`: Fingerprint of the GPG key pair to be used by the server to identify @@ -34,16 +78,26 @@ sections and items are known: password in encrypted format. - `keyring`: Name of the keyring in which authentication data is stored. - `realm`: Name of the realm to use within the digest authentication. -- `import`: Configuration of the file-specific import. +- `import`: Configuration of the file-specific import. Normally, uploaded files + are simply placed in subdirectories below the current working directory; in + nesting order 'upload', the login name of the client and the current date. + For a specific file name, an import script can be started to load a database + from scratch with data from an uploaded dump file. - `database`: The name of the database to import dumps into. Provided as a third parameter to the import script; ignored by the standard `import.sh` script since the database name is determined by the organizational user. - - `dump`: Name of the file that is considered for the import script. Other - files do not trigger the import script. - - `path`: Path to the `monetdb-import` repository where further import - scripts are located. The `Scripts` directory within this repository is used - as working directory for the import script. - - `script`: Path to the script to run when a specific dump file is uploaded. + - `dump`: Name of the uploaded file that is considered for the import script. + Other files do not trigger the import script. The standard `import.sh` + script expects there to be a file called `dump.tar.gz`. + - `path`: Path to the + [monetdb-import](https://github.com/grip-on-software/monetdb-import) + repository where further import scripts are located. The `Scripts` + directory within this repository is used as working directory for the + import script. + - `script`: Path to the script to run when a specific dump file is uploaded. + If the script is placed elsewhere than the currrent working directory, use + an absolute path. The standard `import.sh` script performs database + recreation, archive extraction, import, update and schema publication. - `client`: Accepted logins and public key names. Each item has a configuration key which has the login name of a uploader client, and the value is the name registered in the public key that the uploader must provide in order to be @@ -59,9 +113,69 @@ sections and items are known: ## Running +The upload server can be started directly using the following command: + +``` +gros-upload server +``` + +The subcommand takes various options that can be reviewed by using the +`gros-upload server --help` argument, including debugging instances and +different CGI deployment options. Uploads are stored beneath the "upload" +subdirectory structured of the current working directory. + A `gros-uploader.service` file is provided for installing as a systemd service. One can also use the `upload-session.sh` file to start the service within -a GNOME keyring context with pre-set virtual environments, or directly use the -`python upload.py` script to run the server. The script takes various options -that can be reviewed by using the `--help` argument, including debugging -instances. +a GNOME keyring context, preset to store uploads in `/home/upload/upload` and +logs in `/var/log/upload`, using a `virtualenv` setup shared with the +[controller](https://gros.liacs.nl/data-gathering/api.html#controller-api) of +the agent-based data gathering setup. The script requires a password to unlock +the keyring. In combination with the service, a root user needs to input +a keyring password using a systemd Password Agent, for example by running the +`systemd-tty-ask-password-agent` command, before the server actually starts +under the `upload` user. Some pointers on the advanced setup can be found in +[installation](https://gros.liacs.nl/data-gathering/installation.html#controller) +of the controller environment. + +In order to adjust client authentication credentials, the subcommand +`gros-upload auth [--add|--modify|--delete] --user ... [--password ...]` may be +used. Additional arguments shown in `gros-upload auth --help` allow setting the +secret Digest token and the private key passphrase. Configuring credentials is +also possible for users and the Digest token using the `auth` section and +`secret` option of the `server` section in the [configuration](#configuration) +file, respectively. + +## Development and testing + +To run tests, first install the test dependencies with `make setup_test` which +also installs all dependencies for the upload server. Then `make coverage` +provides test results in the output and in XML versions compatible with, e.g., +JUnit and SonarQube available in the `test-reports/` directory. If you do not +need XML outputs, then run `make test` to just report on test successes and +failures or `make cover` to also have the terminal report on hits and misses in +statements and branches. + +[GitHub Actions](https://github.com/grip-on-software/upload/actions) is used to +run the unit tests and report on coverage on commits and pull requests. This +includes quality gate scans tracked by +[SonarCloud](https://sonarcloud.io/project/overview?id=grip-on-software_upload) +and [Coveralls](https://coveralls.io/github/grip-on-software/upload) for +coverage history. + +The Python module conforms to code style and typing standards which can be +checked using Pylint with `make pylint` and mypy with `make mypy`, after +installing the pylint and mypy dependencies using `make setup_analysis`; typing +reports are XML formats compatible with JUnit and SonarQube placed in the +`mypy-report/` directory. To also receive the HTML report, use `make mypy_html` +instead. + +We publish releases to [PyPI](https://pypi.org/project/gros-upload/) using +`make setup_release` to install dependencies and `make release` which performs +multiple checks: unit tests, typing, lint and version number consistency. The +release files are also published on +[GitHub](https://github.com/grip-on-software/upload/releases) and from there +are archived on [Zenodo](https://zenodo.org/doi/10.5281/zenodo.12784819). + +## License + +GROS encrypted file upload server is licensed under the Apache 2.0 License. diff --git a/encrypted_upload/__init__.py b/encrypted_upload/__init__.py new file mode 100644 index 0000000..1b5a07f --- /dev/null +++ b/encrypted_upload/__init__.py @@ -0,0 +1,24 @@ +""" +Package for PGP upload server. + +Copyright 2017-2020 ICTU +Copyright 2017-2022 Leiden University +Copyright 2017-2024 Leon Helwerda + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from typing import List + +__all__: List[str] = [] +__version__ = "0.0.3" diff --git a/encrypted_upload/__main__.py b/encrypted_upload/__main__.py new file mode 100644 index 0000000..044700b --- /dev/null +++ b/encrypted_upload/__main__.py @@ -0,0 +1,65 @@ +""" +Entry point for the encrypted upload server and associated subcommands. + +Copyright 2017-2020 ICTU +Copyright 2017-2022 Leiden University +Copyright 2017-2024 Leon Helwerda + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from argparse import ArgumentParser, Namespace +from configparser import RawConfigParser +from functools import partial +from . import auth, bootstrap + +def parse_args(config: RawConfigParser) -> Namespace: + """ + Parse command line arguments for subcommands. + """ + + parser = ArgumentParser(description='Encrypted upload server and tools') + subparsers = parser.add_subparsers(title='Subcommands', + description='Server and related tools', + help='Select server or tool to run', + required=True) + + server = subparsers.add_parser('server', + description='Run upload listener server', + help='Run upload listener') + bootstrap.add_args(server, config) + server.set_defaults(callback=partial(bootstrap.bootstrap, config)) + + modify = subparsers.add_parser('auth', + description='Modify keyring credentials', + help='Add, edit or remove authentication') + auth.add_args(modify, config) + modify.set_defaults(callback=auth.handle_command) + + return parser.parse_args() + +def main() -> None: + """ + Main entry point. + """ + + config = RawConfigParser() + config.read('upload.cfg') + args = parse_args(config) + if not callable(args.callback): + raise KeyError('No valid callback specified for subcommand') + + args.callback(args) + +if __name__ == '__main__': + main() diff --git a/encrypted_upload/application.py b/encrypted_upload/application.py new file mode 100644 index 0000000..9192f43 --- /dev/null +++ b/encrypted_upload/application.py @@ -0,0 +1,235 @@ +""" +Listener server which accepts uploaded PGP-encrypted files. + +Copyright 2017-2020 ICTU +Copyright 2017-2022 Leiden University +Copyright 2017-2024 Leon Helwerda + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from argparse import Namespace +from configparser import RawConfigParser +import datetime +import json +from pathlib import Path +import shutil +from subprocess import Popen +import tempfile +from typing import Any, BinaryIO, Dict, List, Optional, Union, TYPE_CHECKING +import cherrypy +import cherrypy.daemon +from cherrypy._cpreqbody import Part +import gpg +from gpg_exchange import Exchange +import keyring +from . import __version__ as VERSION +if TYPE_CHECKING: + from gpg_exchange.exchange import Passphrase +else: + Passphrase = Any + +class Upload: + """ + Upload listener. + """ + + PGP_ARMOR_MIME = "application/pgp-encrypted" + PGP_BINARY_MIME = "application/x-pgp-encrypted-binary" + PGP_ENCRYPT_SUFFIX = ".gpg" + + def __init__(self, args: Namespace, config: RawConfigParser): + self.args = args + self.config = config + + self._keyring = '' + passphrase: Optional[Passphrase] = None + + if self.args.keyring: + self._keyring = str(self.args.keyring) + if self.args.loopback: + passphrase = self._get_passphrase + + self._gpg = Exchange(engine_path=self.args.engine, + passphrase=passphrase) + + def _get_passphrase(self, hint: str, desc: str, prev_bad: int, + hook: Optional[Any] = None) -> str: + # pylint: disable=unused-argument + return keyring.get_password(f'{self._keyring}-secret', 'privkey') + + @cherrypy.expose + @cherrypy.tools.json_in() + @cherrypy.tools.json_out() + def exchange(self) -> Dict[str, str]: + """ + Exchange public keys. + """ + + data = cherrypy.request.json + if not isinstance(data, dict): + raise ValueError('Must provide a JSON object') + if 'pubkey' not in data: + raise ValueError('Must provide a pubkey') + pubkey = str(data['pubkey']) + + temp_dir = tempfile.mkdtemp() + with Exchange(home_dir=temp_dir, engine_path=self.args.engine) as import_gpg: + try: + key = import_gpg.import_key(pubkey)[0] + login = cherrypy.request.login + if login and self.config.has_option('client', login): + if key.uids[0].name != self.config.get('client', login): + raise ValueError('Public key must match client') + + if key.uids[0].name not in self.args.accepted_keys: + raise ValueError('Must be an acceptable public key') + finally: + # Clean up temporary directory + shutil.rmtree(temp_dir) + + # Actual import + client_key = self._gpg.import_key(pubkey)[0] + + # Retrieve our own GPG key and encrypt it with the client key so that + # it cannot be intercepted by others (and thus others cannot send + # encrypted files in name of the client). + server_key = self._gpg.export_key(str(self.args.key)) + ciphertext = self._gpg.encrypt_text(server_key, client_key, + always_trust=True) + + return { + 'pubkey': ciphertext.decode('utf-8') \ + if isinstance(ciphertext, bytes) else ciphertext + } + + def _upload_gpg_file(self, input_file: Optional[BinaryIO], path: Path, + binary: Optional[bool] = None, + passphrase: Optional[Passphrase] = None) -> None: + if input_file is None: + raise ValueError(f'No upload file for {path}') + + try: + with open(path, 'wb') as output_file: + self._gpg.decrypt_file(input_file, output_file, + armor=binary, passphrase=passphrase) + except (gpg.errors.GpgError, ValueError) as error: + # Write the (possibly encrypted) data to a separate file + with open(f"{path}.enc", 'wb') as output_file: + input_file.seek(0) + buf = b'\0' + while buf: + buf = input_file.read(1024) + if buf: + output_file.write(buf) + + raise ValueError(f'Decryption to {path} failed: {error}') from error + + @staticmethod + def _extract_filename(index: int, upload_file: Part) -> str: + if upload_file.filename is None: + raise ValueError(f'No filename provided for file #{index}') + + name = upload_file.filename.split('/')[-1] + if name == '': + raise ValueError(f'No name provided for file #{index}') + + return name + + @classmethod + def _extract_binary_mime(cls, upload_file: Part) -> Optional[bool]: + if upload_file.content_type is None: + return None + + if upload_file.content_type.value == cls.PGP_ARMOR_MIME: + return False + if upload_file.content_type.value == cls.PGP_BINARY_MIME: + return True + + return None + + @cherrypy.expose + @cherrypy.tools.json_out() + def upload(self, files: Optional[Union[Part, List[Part]]] = None) \ + -> Dict[str, bool]: + """ + Perform an upload and import of GPG-encrypted files from a client + which has performed a key exchange. + """ + + if files is None: + raise ValueError('Files required') + if not isinstance(files, list): + files = [files] + + login = str(cherrypy.request.login) + date = datetime.datetime.now().strftime('%Y-%m-%d') + directory = Path(self.args.upload_path) / login / date + directory.mkdir(mode=0o770, parents=True, exist_ok=True) + + for index, upload_file in enumerate(files): + name = self._extract_filename(index, upload_file) + passphrase = None + + if name.endswith(self.PGP_ENCRYPT_SUFFIX): + name = name[:-len(self.PGP_ENCRYPT_SUFFIX)] + if self._keyring: + passphrase = \ + keyring.get_password(f'{self._keyring}-symmetric', + login) + else: + passphrase = self.config['symm'][login] + if name not in self.args.accepted_files: + raise ValueError(f'File #{index}: name {name} is unacceptable') + + binary = self._extract_binary_mime(upload_file) + + try: + self._upload_gpg_file(upload_file.file, directory / name, + binary=binary, passphrase=passphrase) + except ValueError as error: + raise ValueError(f'File {name}: {error}') from error + if name == self.args.import_dump: + process_args: List[str] = [ + '/bin/bash', self.args.import_script, login, date, + self.args.database + ] + path = Path(self.args.import_path) / 'Scripts' + with Popen(process_args, stdout=None, stderr=None, cwd=path): + # Let the import process run but no longer care about it. + pass + + return { + 'success': True + } + + @classmethod + def json_error(cls, status: str, message: str, traceback: str, + version: str) -> str: + """ + Handle HTTP errors by formatting the exception details as JSON. + """ + + cherrypy.response.headers['Content-Type'] = 'application/json' + return json.dumps({ + 'success': False, + 'error': { + 'status': status, + 'message': message, + 'traceback': traceback if cherrypy.request.show_tracebacks else None + }, + 'version': { + 'upload': VERSION, + 'cherrypy': version + } if cherrypy.request.show_tracebacks else {} + }) diff --git a/auth.py b/encrypted_upload/auth.py similarity index 50% rename from auth.py rename to encrypted_upload/auth.py index 6865d4c..66415e2 100644 --- a/auth.py +++ b/encrypted_upload/auth.py @@ -1,9 +1,9 @@ """ -Add, modify or delete client authentication. +Subcommand to add, modify or delete client authentication. Copyright 2017-2020 ICTU Copyright 2017-2022 Leiden University -Copyright 2017-2023 Leon Helwerda +Copyright 2017-2024 Leon Helwerda Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -18,33 +18,18 @@ limitations under the License. """ -import argparse -import configparser +from argparse import ArgumentParser, Namespace +from configparser import RawConfigParser from getpass import getpass -from hashlib import md5 import keyring +from .hash import ha1_nonce -def md5_hex(nonce): +def add_args(parser: ArgumentParser, config: RawConfigParser) -> None: """ - Encode as MD5. + Add command line arguments to an argument parser. """ - return md5(nonce.encode('ISO-8859-1')).hexdigest() - -def ha1_nonce(username, realm, password): - """ - Create an encoded variant for the user's password in the realm. - """ - - return md5_hex(f'{username}:{realm}:{password}') - -def parse_args(config): - """ - Parse command line arguments. - """ - - parser = argparse.ArgumentParser(description='Modify client authentication') - options = parser.add_mutually_exclusive_group() + options = parser.add_mutually_exclusive_group(required=True) options.add_argument('--add', action='store_true', help='Add new user') options.add_argument('--modify', action='store_true', help='Alter user') options.add_argument('--delete', action='store_true', help='Remove user') @@ -60,9 +45,8 @@ def parse_args(config): parser.add_argument('--user', help='Username to modify') parser.add_argument('--password', help='New password or secret to set') - return parser.parse_args() - -def get_password(args, hashed=True, prompt='New password: '): +def get_password(args: Namespace, hashed: bool = True, + prompt: str = 'New password: ') -> str: """ Retrieve the password to be set. """ @@ -73,42 +57,31 @@ def get_password(args, hashed=True, prompt='New password: '): return ha1_nonce(args.user, args.realm, getpass(prompt)) - return args.password if args.password is not None else getpass(prompt) + return str(args.password) if args.password is not None else getpass(prompt) -def main(): +def handle_command(args: Namespace) -> None: """ - Main entry point. + Perform a modification to the authentication keyring. """ - config = configparser.RawConfigParser() - config.read('upload.cfg') - args = parse_args(config) - + domain = str(args.keyring) if args.secret: - keyring.set_password(f'{args.keyring}-secret', 'server', - get_password(args, hashed=False, prompt='Secret key: ')) + keyring.set_password(f'{domain}-secret', 'server', + get_password(args, hashed=False, + prompt='Secret key: ')) elif args.private: - keyring.set_password(f'{args.keyring}-secret', 'privkey', - get_password(args, hashed=False, prompt='Passphrase: ')) + keyring.set_password(f'{domain}-secret', 'privkey', + get_password(args, hashed=False, + prompt='Passphrase: ')) else: - exists = keyring.get_password(args.keyring, args.user) - if args.delete: - if exists: - raise KeyError(f'User {args.user} already exists') - - keyring.delete_password(args.realm, args.user) - elif args.add: - password = get_password(args) - if exists: - raise KeyError(f'User {args.user} already exists') + user = str(args.user) + exists = keyring.get_password(domain, user) + if args.add == bool(exists): + raise KeyError(f'"{user}" {"must" if exists else "does"} not exist') - keyring.set_password(args.keyring, args.user, password) - elif args.modify: + if args.delete: + keyring.delete_password(domain, user) + else: + # Add or modify (after existence check) password = get_password(args) - if not exists: - raise KeyError(f'User {args.user} does not exist') - - keyring.set_password(args.keyring, args.user, password) - -if __name__ == "__main__": - main() + keyring.set_password(domain, user, password) diff --git a/encrypted_upload/bootstrap.py b/encrypted_upload/bootstrap.py new file mode 100644 index 0000000..881c41b --- /dev/null +++ b/encrypted_upload/bootstrap.py @@ -0,0 +1,171 @@ +""" +Module for bootstrapping the encrypted upload server. + +Copyright 2017-2020 ICTU +Copyright 2017-2022 Leiden University +Copyright 2017-2024 Leon Helwerda + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from argparse import ArgumentParser, Namespace +from configparser import RawConfigParser +from pathlib import Path +from typing import Callable +import cherrypy +import keyring +from . import __version__ as VERSION +from .application import Upload +from .hash import ha1_nonce + +def get_ha1_keyring(name: str) -> Callable[[str, str], str]: + """ + Retrieve a function that provides an encoded variable containing the + username, realm and password for digest authentication. The `name` is + the keyring collection name. + """ + + def get_ha1(_: str, username: str) -> str: + """ + Retrieve the HA1 variable for a username from the keyring. + """ + + return str(keyring.get_password(name, username)) + + return get_ha1 + +def add_args(parser: ArgumentParser, config: RawConfigParser) -> None: + """ + Add command line arguments for the server to an argument parser. + """ + + work_dir = Path.cwd() + parser.add_argument('--debug', action='store_true', default=False, + help='Output traces on web') + parser.add_argument('--listen', default=None, + help='Bind address (default: 0.0.0.0, 127.0.0.1 in debug)') + parser.add_argument('--port', default=9090, type=int, + help='Port to listen to (default: 9090)') + parser.add_argument('--log-path', dest='log_path', default=str(work_dir), + help='Path to store logs at in production') + parser.add_argument('--daemonize', action='store_true', default=False, + help='Run the server as a daemon') + parser.add_argument('--pidfile', help='Store process ID in file') + + parser.add_argument('--engine', default=config['server']['engine'], + help='GPG engine path') + parser.add_argument('--upload-path', dest='upload_path', + default=str(work_dir / 'upload'), + help='Upload path') + parser.add_argument('--accepted-files', dest='accepted_files', nargs='*', + default=config['server']['files'].split(' '), + type=set, help='List of filenames allowed for upload') + parser.add_argument('--database', default=config['import']['database'], + help='Database host to import dumps into') + parser.add_argument('--import-dump', default=config['import']['dump'], + dest='import_dump', help='File to import to a database') + parser.add_argument('--import-path', default=config['import']['path'], + dest='import_path', help='Path to the MonetDB importer') + parser.add_argument('--import-script', default=config['import']['script'], + dest='import_script', help='Path to the import script') + parser.add_argument('--key', default=config['server']['key'], + help='Fingerprint of server key pair') + parser.add_argument('--keyring', default=config['server']['keyring'], + help='Name of keyring containing authentication') + parser.add_argument('--realm', default=config['server']['realm'], + help='Name of Digest authentication realm') + parser.add_argument('--accepted-keys', dest='accepted_keys', nargs='*', + default=set(config['client'].values()), + type=set, help='List of accepted names for public keys') + parser.add_argument('--loopback', action='store_true', + help='Use loopback pinhole to read passphrase from keyring') + + server = parser.add_mutually_exclusive_group() + server.add_argument('--fastcgi', action='store_true', default=False, + help='Start a FastCGI server instead of HTTP') + server.add_argument('--scgi', action='store_true', default=False, + help='Start a SCGI server instead of HTTP') + server.add_argument('--cgi', action='store_true', default=False, + help='Start a CGI server instead of HTTP') + +def _update_keyring(config: RawConfigParser, args: Namespace, + auth_key: str) -> str: + keyring_name = str(args.keyring) + auth_keyring = keyring.get_password(f'{keyring_name}-secret', 'server') + if auth_keyring is not None: + auth_key = auth_keyring + elif auth_key != '': + keyring.set_password(f'{keyring_name}-secret', 'server', auth_key) + else: + raise ValueError('No server secret auth key provided') + + for user, password in config['auth'].items(): + keyring.set_password(keyring_name, user, + ha1_nonce(user, str(args.realm), password)) + for user, passphrase in config['symm'].items(): + keyring.set_password(f'{keyring_name}-symmetric', user, passphrase) + + return auth_key + +def bootstrap(config: RawConfigParser, args: Namespace) -> None: + """ + Set up the upload server. + """ + + debug = bool(args.debug) + if args.listen is not None: + bind_address = str(args.listen) + elif debug: + bind_address = '127.0.0.1' + else: + bind_address = '0.0.0.0' + + auth_key = str(config['server'].get('secret', '')) + if args.keyring: + auth_key = _update_keyring(config, args, auth_key) + ha1 = get_ha1_keyring(args.keyring) + else: + ha1 = cherrypy.lib.auth_digest.get_ha1_dict_plain(dict(config['auth'])) + + if debug: + server = f'gros-upload/{VERSION} CherryPy/{cherrypy.__version__}' + else: + server = 'gros-upload CherryPy' + + conf = { + 'global': { + }, + '/': { + 'error_page.default': Upload.json_error, + 'response.headers.server': server, + 'tools.auth_digest.on': True, + 'tools.auth_digest.realm': str(args.realm), + 'tools.auth_digest.get_ha1': ha1, + 'tools.auth_digest.key': str(auth_key) + } + } + log_path = Path(args.log_path) + cherrypy.config.update({ + 'server.max_request_body_size': 1000 * 1024 * 1024, + 'server.socket_host': bind_address, + 'server.socket_port': args.port, + 'request.show_tracebacks': debug, + 'log.screen': debug, + 'log.access_file': '' if debug else str(log_path / 'access.log'), + 'log.error_file': '' if debug else str(log_path / 'error.log'), + }) + + # Start the application and server daemon. + cherrypy.tree.mount(Upload(args, config), '/upload', conf) + cherrypy.daemon.start(daemonize=args.daemonize, pidfile=args.pidfile, + fastcgi=args.fastcgi, scgi=args.scgi, cgi=args.cgi) diff --git a/encrypted_upload/hash.py b/encrypted_upload/hash.py new file mode 100644 index 0000000..ebffc8f --- /dev/null +++ b/encrypted_upload/hash.py @@ -0,0 +1,35 @@ +""" +Digest hash functions. + +Copyright 2017-2020 ICTU +Copyright 2017-2022 Leiden University +Copyright 2017-2024 Leon Helwerda + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from hashlib import md5 + +def md5_hex(nonce: str) -> str: + """ + Encode as MD5. + """ + + return md5(nonce.encode('ISO-8859-1')).hexdigest() + +def ha1_nonce(username: str, realm: str, password: str) -> str: + """ + Create an encoded variant for the user's password in the realm. + """ + + return md5_hex(f'{username}:{realm}:{password}') diff --git a/import.sh b/import.sh index 38d6f7c..2707e2e 100755 --- a/import.sh +++ b/import.sh @@ -4,7 +4,7 @@ # # Copyright 2017-2020 ICTU # Copyright 2017-2022 Leiden University -# Copyright 2017-2023 Leon Helwerda +# Copyright 2017-2024 Leon Helwerda # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -20,6 +20,11 @@ set -e +if [ -z "$1" ]; then + echo "Usage: ./import.sh [dbhost]" >&2 + echo "Recreate database from dump.tar.gz import, with update and schema" +fi + ORGANIZATION=$1 shift DATE=$1 @@ -42,12 +47,15 @@ fi DB="gros_$ORGANIZATION" cd "$IMPORTER/Scripts" +# Create database python "recreate_database.py" --force --no-table-import --no-schema --keep-jenkins -h "$HOST" -d "$DB" +# Extract dump file if [ ! -d "$DIRECTORY/dump" ]; then tar --directory "$DIRECTORY" --no-same-owner --no-same-permissions -xzf "$DIRECTORY/dump.tar.gz" fi +# Import table dumps from CSV/SQL files set +e "./import_tables.sh" "$HOST" "$DIRECTORY/dump/gros-$DATE" "$DB" status=$? @@ -55,11 +63,16 @@ set -e if [ $status -ne 0 ]; then echo "Failed to import all tables correctly" >&2 else + # Update imported database to current state python "update_database.py" -h "$HOST" -d "$DB" fi -cp "$DIRECTORY/dump/tables-documentation.json" $SCHEMA -cp "$DIRECTORY/dump/tables-schema.json" $SCHEMA +# Export schema files for further publication/comparison +if [ -d $SCHEMA ]; then + cp "$DIRECTORY/dump/tables-documentation.json" $SCHEMA + cp "$DIRECTORY/dump/tables-schema.json" $SCHEMA +fi +# Remove extracted directory rm -rf "$DIRECTORY/dump" exit $status diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..c741e4e --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,60 @@ +[project] +name = "gros-upload" +version = "0.0.3" +description = "Encrypted file upload server" +readme = "README.md" +authors = [{name = "Leon Helwerda", email = "l.s.helwerda@liacs.leidenuniv.nl"}] +license = {text = "Apache 2.0"} +requires-python = ">=3.8" +dependencies = [ + "CherryPy==18.9.0", + "gpg_exchange==0.0.7", + "keyring==25.2.1" +] +classifiers=[ + "Development Status :: 3 - Alpha", + "Environment :: Console", + "Environment :: Web Environment", + "Intended Audience :: Developers", + "Topic :: Communications", + "Topic :: Internet :: WWW/HTTP :: HTTP Servers", + "Topic :: Security :: Cryptography", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", +] +keywords = ["gpg encryption", "database importer", "secure upload"] + +[project.scripts] +gros-upload = "encrypted_upload.__main__:main" + +[project.urls] +"Homepage" = "https://gros.liacs.nl" +"PyPI" = "https://pypi.python.org/pypi/gros-upload" +"Source Code" = "https://github.com/grip-on-software/upload" +"Issues" = "https://github.com/grip-on-software/upload/issues" +"Pull Requests" = "https://github.com/grip-on-software/upload/pulls" +"CI: GitHub Actions" = "https://github.com/grip-on-software/upload/actions" +"CI: Coveralls" = "https://coveralls.io/github/grip-on-software/upload?branch=master" +"CI: SonarCloud" = "https://sonarcloud.io/project/overview?id=grip-on-software_upload" + +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" + +[tool.setuptools] +packages = ["encrypted_upload"] + +[tool.setuptools.package-data] +"encrypted_upload" = ["py.typed"] + +[tool.mypy] +mypy_path = "typeshed" + +[tool.pytest.ini_options] +testpaths = "test" +python_files = "*.py" diff --git a/requirements-analysis.txt b/requirements-analysis.txt new file mode 100644 index 0000000..8a05eab --- /dev/null +++ b/requirements-analysis.txt @@ -0,0 +1,3 @@ +pylint==3.2.2 +lxml==5.2.2 +mypy==1.10.0 diff --git a/requirements-release.txt b/requirements-release.txt new file mode 100644 index 0000000..c1be97f --- /dev/null +++ b/requirements-release.txt @@ -0,0 +1,4 @@ +setuptools==70.0.0 +build==1.2.1 +wheel==0.43.0 +twine==5.1.1 diff --git a/requirements-test.txt b/requirements-test.txt new file mode 100644 index 0000000..1ca9ba6 --- /dev/null +++ b/requirements-test.txt @@ -0,0 +1,5 @@ +# Install dependencies for all modules +-r requirements.txt +# Test dependencies +coverage==7.4.4 +pytest==8.2.2 diff --git a/requirements.txt b/requirements.txt index 7bb9627..d0dc53a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,3 @@ -configparser -cherrypy -gpg_exchange>=0.0.7 -keyring +CherryPy==18.9.0 +gpg_exchange==0.0.7 +keyring==25.2.1 diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..980cab0 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[mypy] +mypy_path = typeshed diff --git a/sonar-project.properties b/sonar-project.properties new file mode 100644 index 0000000..7eae676 --- /dev/null +++ b/sonar-project.properties @@ -0,0 +1,19 @@ +# must be unique in a given SonarQube instance +sonar.projectKey=grip-on-software_upload +sonar.organization=grip-on-software +# this is the name and version displayed in the SonarQube UI. Was mandatory prior to SonarQube 6.1. +sonar.projectName=Uploader +sonar.projectVersion=0.0.3 + +# Path is relative to the sonar-project.properties file. Replace "\" by "/" on Windows. +# This property is optional if sonar.modules is set. +sonar.sources=encrypted_upload +sonar.tests=test +sonar.python.version=3 +sonar.python.pylint.reportPaths=pylint-report.txt +sonar.python.xunit.reportPath=*-report/TEST-*.xml +sonar.python.xunit.skipDetails=true +sonar.python.coverage.reportPaths=*-report*/cobertura.xml + +# Encoding of the source code. Default is default system encoding +#sonar.sourceEncoding=UTF-8 diff --git a/test/__init__.py b/test/__init__.py new file mode 100644 index 0000000..9d7b3a4 --- /dev/null +++ b/test/__init__.py @@ -0,0 +1,19 @@ +""" +Test package for PGP upload server. + +Copyright 2017-2020 ICTU +Copyright 2017-2022 Leiden University +Copyright 2017-2024 Leon Helwerda + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" diff --git a/test/application.py b/test/application.py new file mode 100644 index 0000000..e938fba --- /dev/null +++ b/test/application.py @@ -0,0 +1,334 @@ +""" +Tests for listener server which accepts uploaded PGP-encrypted files. + +Copyright 2017-2020 ICTU +Copyright 2017-2022 Leiden University +Copyright 2017-2023 Leon Helwerda + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from argparse import Namespace +from configparser import RawConfigParser +from datetime import datetime +from email.message import EmailMessage +from email.policy import HTTP +import json +import os +from pathlib import Path +import shutil +import tempfile +from typing import Any, Dict, List, Literal, Optional, Tuple, Union, overload +from unittest.mock import MagicMock, patch +import cherrypy +from cherrypy.test import helper +from gpg_exchange import Exchange +from encrypted_upload.application import Upload + +class UploadTest(helper.CPWebCase): + """ + Tests for upload listener. + """ + + server_key_fpr: str = 'MISSING' + server_pubkey: Union[str, bytes] + client_key_fpr: str = 'MISSING' + client_pubkey: Union[str, bytes] + temp_dir: str + gpg: Exchange + client_gpg: Exchange + get_password: MagicMock + popen: MagicMock + + @classmethod + def _get_passphrase(cls, hint: str, desc: str, prev_bad: int, + hook: Optional[Any] = None) -> str: + # pylint: disable=unused-argument + return 'pass' + + @classmethod + def setup_server(cls) -> None: + """ + Set up the application server. + """ + + # Pre-generate a key so we can actually make proper encrypted responses + cls.gpg = Exchange(passphrase=cls._get_passphrase) + try: + cls.server_key_fpr = cls.gpg.find_key('GROS upload server test').fpr + except KeyError: + key = cls.gpg.generate_key('GROS upload server test', + 'upload@gros.test', + comment='GROS upload server key') + cls.server_key_fpr = key.fpr + cls.server_pubkey = cls.gpg.export_key(cls.server_key_fpr) + + cls.temp_dir = tempfile.mkdtemp() + cls.client_gpg = Exchange(home_dir=cls.temp_dir, + passphrase=cls._get_passphrase) + try: + cls.client_key_fpr = cls.client_gpg.find_key('GROS TEST').fpr + except KeyError: + key = cls.client_gpg.generate_key('GROS TEST', 'test@gros.test', + comment='GROS upload client key') + cls.client_key_fpr = key.fpr + cls.client_pubkey = cls.client_gpg.export_key(cls.client_key_fpr) + cls.client_gpg.import_key(cls.server_pubkey) + + keyring_patcher = patch('keyring.get_password', return_value='pass') + cls.get_password = keyring_patcher.start() + cls.addClassCleanup(keyring_patcher.stop) + + subprocess_patcher = patch('encrypted_upload.application.Popen') + cls.popen = subprocess_patcher.start() + cls.addClassCleanup(subprocess_patcher.stop) + + args = Namespace() + args.engine = None + args.upload_path = 'test/sample/upload' + args.accepted_files = ('dump.tar.gz', 'message.txt') + args.database = 'localhost' + args.import_dump = 'dump.tar.gz' + args.import_path = 'test/sample' + args.import_script = 'import.sh' + args.key = cls.server_key_fpr + args.keyring = 'gros-uploader' + args.realm = 'upload' + args.accepted_keys = ('GROS TEST', 'GROS EX') + args.loopback = True + + config = RawConfigParser() + config['client'] = {} + config['client']['test'] = 'GROS TEST' + config['symm'] = {} + config['symm']['test'] = 'pass' + + cherrypy.tree.mount(Upload(args, config), '/upload', { + 'global': {}, + '/': { + 'error_page.default': Upload.json_error + } + }) + + @classmethod + def tearDownClass(cls) -> None: + try: + cls.gpg.delete_key(cls.server_key_fpr, secret=True) + except KeyError: # pragma: no cover + pass + del cls.gpg + + try: + cls.client_gpg.delete_key(cls.client_key_fpr, secret=True) + except KeyError: # pragma: no cover + pass + try: + cls.client_gpg.delete_key(cls.server_key_fpr) + except KeyError: # pragma: no cover + pass + del cls.client_gpg + shutil.rmtree(cls.temp_dir) + + def tearDown(self) -> None: + try: + self.gpg.delete_key(self.client_key_fpr) + except KeyError: # pragma: no cover + pass + + def perform_exchange_request(self, data: Dict[str, str]) -> None: + """ + Perform a request to the exchange JSON endpoint. + """ + + body = json.dumps(data) + self.getPage('/upload/exchange', + headers=[('Content-Type', 'application/json'), + ('Content-Length', str(len(body)))], + method='POST', body=body) + + @overload + def check_json(self, key: str) -> str: + ... + + @overload + def check_json(self, key: Literal[True]) -> None: + ... + + def check_json(self, key) -> Optional[str]: + """ + Check if the response headers/body indicate correct JSON and has proper + object keys. For the 'error' key, we check additional error response + structure, namely the 'success' key; the error traceback is returned. + If `key` is True, then the 'success' key is checked for the non-error + response. Otherwise, the actual value of the object key is returned. + """ + + self.assertHeader('Content-Type', 'application/json') + data: Dict[str, Union[Dict[str, str], str]] = json.loads(self.body) + self.assertIn('success' if key is True else key, data) + if key == 'error': + self.assertFalse(data['success']) + error = data[key] + if not isinstance(error, dict): + raise AssertionError(f'Expected object data for {key}: {error}') + actual = error['traceback'] + elif key is True: + self.assertTrue(data['success']) + return None + else: + value = data[key] + if not isinstance(value, str): + raise AssertionError(f'Expected string data for {key}: {value}') + actual = value + + return actual + + def test_exchange(self) -> None: + """ + Test exchanging public keys. + """ + + self.perform_exchange_request({ + 'pubkey': self.client_pubkey.decode('utf-8') \ + if isinstance(self.client_pubkey, bytes) else self.client_pubkey + }) + pubkey = self.check_json('pubkey') + self.assertEqual(self.client_gpg.decrypt_text(pubkey.encode('utf-8'), + verify=False), + self.server_pubkey) + self.assertIsNotNone(self.gpg.find_key(self.client_key_fpr)) + + # Unknown public keys are not accepted. + with open('test/sample/other.gpg', encoding='utf-8') as pubkey_file: + other_pubkey = pubkey_file.read() + + self.perform_exchange_request({ + 'pubkey': other_pubkey + }) + self.assertIn('Must be an acceptable public key', + self.check_json('error')) + + # Provide invalid JSON objects + self.getPage('/upload/exchange', method='POST', + headers=[('Content-Type', 'application/json'), + ('Content-Length', '2')], + body=json.dumps([])) + self.assertIn('Must provide a JSON object', self.check_json('error')) + + self.perform_exchange_request({}) + self.assertIn('Must provide a pubkey', self.check_json('error')) + + def _encrypt_upload(self, path_name: str, upload_name: Optional[str], + armor: bool, message: EmailMessage) -> bytes: + key = self.client_gpg.find_key(self.server_key_fpr) + mime_type = Upload.PGP_ARMOR_MIME if armor else Upload.PGP_BINARY_MIME + with open(f'test/sample/{path_name}', 'rb') as upload_file: + if upload_name is None: + # Keep unencrypted + upload_name = path_name + encrypted_payload = upload_file.read() + else: + with tempfile.TemporaryFile() as temp_file: + self.client_gpg.encrypt_file(upload_file, temp_file, key, + always_trust=True, armor=armor) + temp_file.seek(0, os.SEEK_SET) + encrypted_payload = temp_file.read() + + form_data = EmailMessage(policy=HTTP) + form_data.add_header('Content-Type', mime_type) + form_data.add_header('Content-Disposition', 'form-data', + name='files', filename=upload_name) + form_data.set_payload(encrypted_payload) + message.attach(form_data) + + return encrypted_payload + + def _make_upload_body(self, message: EmailMessage, + payloads: List[bytes]) -> bytes: + encoded_parts = message.as_bytes().split(b'\r\n\r\n', maxsplit=1)[1] + boundary = message.get_boundary('1234567890').encode('ascii') + body = b'--' + boundary + for part, payload in zip(encoded_parts.split(b'--' + boundary)[1:-1], + payloads): + body += part.split(b'\r\n\r\n', maxsplit=1)[0] + \ + b'\r\n\r\n' + payload + b'\r\n--' + boundary + body += b'--' + return body + + def perform_upload_request(self, *files: str, + names: Optional[Tuple[Optional[str], ...]] = None, + armor: bool = False) -> None: + """ + Perform a request to the upload endpoint. + """ + + if names is None: + names = files + + message = EmailMessage(policy=HTTP) + message.add_header('Content-Type', 'multipart/form-data') + payloads = [] + for path_name, upload_name in zip(files, names): + encrypted_payload = self._encrypt_upload(path_name, upload_name, + armor, message) + payloads.append(encrypted_payload) + + body = self._make_upload_body(message, payloads) + self.getPage('/upload/upload', method='POST', + headers=[('Content-Type', message['Content-Type']), + ('Content-Length', str(len(body)))], + body=body) + + def test_upload(self) -> None: + """ + Test performing an upload and import of GPG-encrypted files. + """ + + date = datetime.now().strftime('%Y-%m-%d') + + # Pretend an exchange has taken place. + self.gpg.import_key(self.client_pubkey) + + self.perform_upload_request('message.txt') + self.check_json(True) + upload_path = Path(f'test/sample/upload/None/{date}/message.txt') + self.assertTrue(upload_path.exists()) + + self.perform_upload_request('dump.tar.gz', armor=True) + self.check_json(True) + self.popen.assert_called_once_with([ + '/bin/bash', 'import.sh', 'None', date, 'localhost' + ], stdout=None, stderr=None, cwd=Path('test/sample/Scripts')) + + # Zero files + self.getPage('/upload/upload', method='POST') + self.assertIn('Files required', self.check_json('error')) + + # Multiple files + self.perform_upload_request('message.txt', 'dump.tar.gz', + names=('message.txt.gpg', 'dump.tar.gz')) + self.check_json(True) + + # Files without names + self.perform_upload_request('message.txt', names=('/',)) + self.assertIn('No name provided for file #0', self.check_json('error')) + + # Unaccepted filenames + self.perform_upload_request('message.txt', names=('other.txt',)) + self.assertIn('File #0: name other.txt is unacceptable', + self.check_json('error')) + + # Decryption problems + self.perform_upload_request('message.txt', names=(None,)) + self.assertRegex(self.check_json('error'), + r'.*File message\.txt: Decryption to .* failed: .*') diff --git a/test/auth.py b/test/auth.py new file mode 100644 index 0000000..5bcb399 --- /dev/null +++ b/test/auth.py @@ -0,0 +1,126 @@ +""" +Tests for subcommand to add, modify or delete client authentication. + +Copyright 2017-2020 ICTU +Copyright 2017-2022 Leiden University +Copyright 2017-2024 Leon Helwerda + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from argparse import ArgumentParser +from configparser import RawConfigParser +import unittest +from unittest.mock import patch, MagicMock +from encrypted_upload.auth import add_args, get_password, handle_command +from encrypted_upload.hash import ha1_nonce + +class AuthTest(unittest.TestCase): + """ + Tests for authentication subcommand. + """ + + def setUp(self) -> None: + self.parser = ArgumentParser() + self.config = RawConfigParser() + self.config.read('upload.cfg.example') + + def test_add_args(self) -> None: + """ + Test adding command line arguments. + """ + + add_args(self.parser, self.config) + args = self.parser.parse_args(['--add']) + self.assertTrue(args.add) + self.assertFalse(args.modify) + + @patch('encrypted_upload.auth.getpass', return_value='pass') + def test_get_password(self, getpass: MagicMock) -> None: + """ + Test retrieving the password to be set. + """ + + add_args(self.parser, self.config) + args = self.parser.parse_args(['--add', '--user', 'user']) + self.assertEqual(get_password(args, hashed=False), 'pass') + getpass.assert_called_once_with('New password: ') + + getpass.reset_mock() + self.assertEqual(get_password(args, hashed=True, prompt='PWD:'), + ha1_nonce('user', '$SERVER_REALM', 'pass')) + getpass.assert_called_once_with('PWD:') + + getpass.reset_mock() + args = self.parser.parse_args([ + '--add', '--user', 'user', '--realm', 'REALM', '--password', 'other' + ]) + self.assertEqual(get_password(args, hashed=False), 'other') + self.assertEqual(get_password(args, hashed=True), + ha1_nonce('user', 'REALM', 'other')) + getpass.assert_not_called() + + @patch('keyring.get_password', return_value='pass') + @patch('keyring.set_password') + @patch('keyring.delete_password') + def test_handle_command(self, delete_password: MagicMock, + set_password: MagicMock, get: MagicMock) -> None: + """ + Test performin a modification to the keyring. + """ + + add_args(self.parser, self.config) + + args = self.parser.parse_args(['--delete', '--user', 'user']) + handle_command(args) + delete_password.assert_called_once_with('$SERVER_KEYRING', 'user') + get.return_value = None + with self.assertRaisesRegex(KeyError, '"user" does not exist'): + handle_command(args) + + args = self.parser.parse_args([ + '--add', '--user', 'user', '--password', 'mypass', + '--realm', 'ex', '--keyring', 'ring' + ]) + handle_command(args) + set_password.assert_called_once_with('ring', 'user', + ha1_nonce('user', 'ex', 'mypass')) + get.return_value = 'mypass' + with self.assertRaisesRegex(KeyError, '"user" must not exist'): + handle_command(args) + + set_password.reset_mock() + args = self.parser.parse_args([ + '--modify', '--user', 'user', '--password', 'newpass', + '--realm', 'ex', '--keyring', 'domain' + ]) + handle_command(args) + set_password.assert_called_once_with('domain', 'user', + ha1_nonce('user', 'ex', 'newpass')) + get.return_value = None + with self.assertRaisesRegex(KeyError, '"user" does not exist'): + handle_command(args) + + set_password.reset_mock() + args = self.parser.parse_args([ + '--secret', '--password', 'token', '--keyring', 'domain' + ]) + handle_command(args) + set_password.assert_called_once_with('domain-secret','server', 'token') + + set_password.reset_mock() + args = self.parser.parse_args([ + '--private', '--password', 'priv', '--keyring', 'domain' + ]) + handle_command(args) + set_password.assert_called_once_with('domain-secret', 'privkey', 'priv') diff --git a/test/bootstrap.py b/test/bootstrap.py new file mode 100644 index 0000000..5bfe0fe --- /dev/null +++ b/test/bootstrap.py @@ -0,0 +1,93 @@ +""" +Tests for module that bootstraps the encrypted upload server. + +Copyright 2017-2020 ICTU +Copyright 2017-2022 Leiden University +Copyright 2017-2024 Leon Helwerda + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from argparse import ArgumentParser +from configparser import RawConfigParser +import unittest +from unittest.mock import patch, MagicMock +from encrypted_upload.bootstrap import get_ha1_keyring, add_args, bootstrap + +class BootstrapTest(unittest.TestCase): + """ + Tests for bootstrap methods. + """ + + def setUp(self) -> None: + self.config = RawConfigParser() + self.config.read("upload.cfg.example") + + self.parser = ArgumentParser() + + @patch('keyring.get_password', return_value='pass') + def test_get_ha1_keyring(self, get_password: MagicMock) -> None: + """ + Test providing digest authentication via keyring. + """ + + get_ha1 = get_ha1_keyring('domain') + self.assertEqual(get_ha1('REALM', 'user'), 'pass') + get_password.assert_called_once_with('domain', 'user') + + def test_add_args(self) -> None: + """ + Test adding command line arguments for the server. + """ + + add_args(self.parser, self.config) + args = self.parser.parse_args(['--port', '8080']) + self.assertEqual(args.port, 8080) + + @patch('keyring.get_password', return_value='pass') + @patch('keyring.set_password') + @patch('cherrypy.daemon.start') + def test_bootstrap(self, daemon: MagicMock, set_password: MagicMock, + get_password: MagicMock) -> None: + """ + Test setting up the upload server. + """ + + self.config['server']['keyring'] = '' + add_args(self.parser, self.config) + args = self.parser.parse_args(['--keyring', 'domain', '--daemonize']) + bootstrap(self.config, args) + get_password.assert_called_with('domain-secret', 'server') + set_password.assert_called_with('domain-symmetric', + '$CLIENT_ID'.lower(), + '$CLIENT_PASSPHRASE') + daemon.assert_called_once_with(daemonize=True, pidfile=None, + fastcgi=False, scgi=False, cgi=False) + + get_password.reset_mock() + set_password.reset_mock() + + args = self.parser.parse_args(['--listen', '127.0.0.2', '--debug']) + bootstrap(self.config, args) + get_password.assert_not_called() + set_password.assert_not_called() + + get_password.configure_mock(return_value=None) + args = self.parser.parse_args(['--keyring', 'set', '--debug']) + bootstrap(self.config, args) + set_password.assert_any_call('set-secret', 'server', '$SERVER_SECRET') + + self.config['server']['secret'] = '' + with self.assertRaisesRegex(ValueError, + 'No server secret auth key provided'): + bootstrap(self.config, args) diff --git a/test/hash.py b/test/hash.py new file mode 100644 index 0000000..71dd20c --- /dev/null +++ b/test/hash.py @@ -0,0 +1,44 @@ +""" +Tests for Digest hash functions. + +Copyright 2017-2020 ICTU +Copyright 2017-2022 Leiden University +Copyright 2017-2024 Leon Helwerda + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from hashlib import md5 +import unittest +from encrypted_upload.hash import md5_hex, ha1_nonce + +class HashTest(unittest.TestCase): + """ + Tests for Digest hash functions. + """ + + def test_md5_hex(self) -> None: + """ + Test encoding as MD5. + """ + + self.assertEqual(md5_hex(""), "d41d8cd98f00b204e9800998ecf8427e") + self.assertEqual(md5_hex("test"), md5(b'test').hexdigest()) + + def test_ha1_nonce(self) -> None: + """ + Test creating an encoded variant of a user password in a realm. + """ + + self.assertEqual(ha1_nonce('user', 'realm', 'pass'), + md5(b'user:realm:pass').hexdigest()) diff --git a/test/sample/Scripts/import.sh b/test/sample/Scripts/import.sh new file mode 100755 index 0000000..6f08757 --- /dev/null +++ b/test/sample/Scripts/import.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +# CWD = test/sample/Scripts +echo "SUCCESS" > ../upload/dump.log diff --git a/test/sample/dump.tar.gz b/test/sample/dump.tar.gz new file mode 100644 index 0000000..576c424 Binary files /dev/null and b/test/sample/dump.tar.gz differ diff --git a/test/sample/message.txt b/test/sample/message.txt new file mode 100644 index 0000000..d25f218 --- /dev/null +++ b/test/sample/message.txt @@ -0,0 +1 @@ +Test message diff --git a/test/sample/other.gpg b/test/sample/other.gpg new file mode 100644 index 0000000..a68993e --- /dev/null +++ b/test/sample/other.gpg @@ -0,0 +1,41 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQGNBF2lnPIBDAC5cL9PQoQLTMuhjbYvb4Ncuuo0bfmgPRFywX53jPhoFf4Zg6mv +/seOXpgecTdOcVttfzC8ycIKrt3aQTiwOG/ctaR4Bk/t6ayNFfdUNxHWk4WCKzdz +/56fW2O0F23qIRd8UUJp5IIlN4RDdRCtdhVQIAuzvp2oVy/LaS2kxQoKvph/5pQ/ +5whqsyroEWDJoSV0yOb25B/iwk/pLUFoyhDG9bj0kIzDxrEqW+7Ba8nocQlecMF3 +X5KMN5kp2zraLv9dlBBpWW43XktjcCZgMy20SouraVma8Je/ECwUWYUiAZxLIlMv +9CurEOtxUw6N3RdOtLmYZS9uEnn5y1UkF88o8Nku890uk6BrewFzJyLAx5wRZ4F0 +qV/yq36UWQ0JB/AUGhHVPdFf6pl6eaxBwT5GXvbBUibtf8YI2og5RsgTWtXfU7eb +SGXrl5ZMpbA6mbfhd0R8aPxWfmDWiIOhBufhMCvUHh1sApMKVZnvIff9/0Dca3wb +vLIwa3T4CyshfT0AEQEAAbQhQm9iIEJhYmJhZ2UgPGJvYkBvcGVucGdwLmV4YW1w +bGU+iQHOBBMBCgA4AhsDBQsJCAcCBhUKCQgLAgQWAgMBAh4BAheAFiEE0aZuGiOx +gsmYD3iM+/zIKgFeczAFAl2lnvoACgkQ+/zIKgFeczBvbAv/VNk90a6hG8Od9xTz +XxH5YRFUSGfIA1yjPIVOnKqhMwps2U+sWE3urL+MvjyQRlyRV8oY9IOhQ5Esm6DO +ZYrTnE7qVETm1ajIAP2OFChEc55uH88x/anpPOXOJY7S8jbn3naC9qad75BrZ+3g +9EBUWiy5p8TykP05WSnSxNRt7vFKLfEB4nGkehpwHXOVF0CRNwYle42bg8lpmdXF +DcCZCi+qEbafmTQzkAqyzS3nCh3IAqq6Y0kBuaKLm2tSNUOlZbD+OHYQNZ5Jix7c +ZUzs6Xh4+I55NRWl5smrLq66yOQoFPy9jot/Qxikx/wP3MsAzeGaZSEPc0fHp5G1 +6rlGbxQ3vl8/usUV7W+TMEMljgwd5x8POR6HC8EaCDfVnUBCPi/Gv+egLjsIbPJZ +ZEroiE40e6/UoCiQtlpQB5exPJYSd1Q1txCwueih99PHepsDhmUQKiACszNU+RRo +zAYau2VdHqnRJ7QYdxHDiH49jPK4NTMyb/tJh2TiIwcmsIpGuQGNBF2lnPIBDADW +ML9cbGMrp12CtF9b2P6z9TTT74S8iyBOzaSvdGDQY/sUtZXRg21HWamXnn9sSXvI +DEINOQ6A9QxdxoqWdCHrOuW3ofneYXoG+zeKc4dC86wa1TR2q9vW+RMXSO4uImA+ +Uzula/6k1DogDf28qhCxMwG/i/m9g1c/0aApuDyKdQ1PXsHHNlgd/Dn6rrd5y2AO +baifV7wIhEJnvqgFXDN2RXGjLeCOHV4Q2WTYPg/S4k1nMXVDwZXrvIsA0YwIMgIT +86Rafp1qKlgPNbiIlC1g9RY/iFaGN2b4Ir6GDohBQSfZW2+LXoPZuVE/wGlQ01rh +827KVZW4lXvqsge+wtnWlszcselGATyzqOK9LdHPdZGzROZYI2e8c+paLNDdVPL6 +vdRBUnkCaEkOtl1mr2JpQi5nTU+gTX4IeInC7E+1a9UDF/Y85ybUz8XV8rUnR76U +qVC7KidNepdHbZjjXCt8/Zo+Tec9JNbYNQB/e9ExmDntmlHEsSEQzFwzj8sxH48A +EQEAAYkBtgQYAQoAIBYhBNGmbhojsYLJmA94jPv8yCoBXnMwBQJdpZzyAhsMAAoJ +EPv8yCoBXnMw6f8L/26C34dkjBffTzMj5Bdzm8MtF67OYneJ4TQMw7+41IL4rVcS +KhIhk/3Ud5knaRtP2ef1+5F66h9/RPQOJ5+tvBwhBAcUWSupKnUrdVaZQanYmtSx +cVV2PL9+QEiNN3tzluhaWO//rACxJ+K/ZXQlIzwQVTpNhfGzAaMVV9zpf3u0k14i +tcv6alKY8+rLZvO1wIIeRZLmU0tZDD5HtWDvUV7rIFI1WuoLb+KZgbYn3OWjCPHV +dTrdZ2CqnZbG3SXw6awH9bzRLV9EXkbhIMez0deCVdeo+wFFklh8/5VK2b0vk/+w +qMJxfpa1lHvJLobzOP9fvrswsr92MA2+k901WeISR7qEzcI0Fdg8AyFAExaEK6Vy +jP7SXGLwvfisw34OxuZr3qmx1Sufu4toH3XrB7QJN8XyqqbsGxUCBqWif9RSK4xj +zRTe56iPeiSJJOIciMP9i2ldI+KgLycyeDvGoBj0HCLO3gVaBe4ubVrj5KjhX2PV +NEJd3XZRzaXZE2aAMQ== +=NXei +-----END PGP PUBLIC KEY BLOCK----- diff --git a/typeshed/cherrypy/__init__.pyi b/typeshed/cherrypy/__init__.pyi new file mode 100644 index 0000000..3b9e012 --- /dev/null +++ b/typeshed/cherrypy/__init__.pyi @@ -0,0 +1,64 @@ +from typing import Any, Callable, Dict, List, Literal, Optional, TextIO, \ + TypeVar, Union +from . import dispatch +from . import process +from . import _cptree + +__version__: str + +class HTTPError(Exception): + def __call__(self) -> 'ResponseBody': ... + +class HTTPRedirect(HTTPError): + pass + +class NotFound(HTTPError): + pass + +class Request: + headers: Dict[str, str] = ... + config: Dict[str, Any] = ... + handler: Optional[Callable[[], 'ResponseBody']] = ... + method: str = ... + show_tracebacks: bool = ... + path_info: str = ... + query_string: str = ... + json: Union[List[Any], Dict[str, Any]] = ... + login: Optional[Union[Literal[False], str]] = ... + +class ResponseBody: + pass + +class Response: + body: ResponseBody = ... + status: int = ... + headers: Dict[str, str] = ... + +class Serving: + request: Request = ... + response: Response = ... + +serving: Serving = ... +request: Request = ... +response: Response = ... +session: Dict[str, str] = ... +config: Dict[str, Any] = ... + +engine = process.bus +tree: _cptree.Tree = ... + +Application = TypeVar('Application', bound=object, covariant=True) +ExposedFunction = Callable[[Application], str] +JSONFunction = Callable[[Application], Union[List[Any], Dict[str, Any]]] + +def expose(func: ExposedFunction, alias: Optional[str] = ...) -> ExposedFunction: ... + +class Toolbox: + @staticmethod + def json_in() -> Callable[[ExposedFunction], ExposedFunction]: ... + @staticmethod + def json_out() -> Callable[[JSONFunction], ExposedFunction]: ... + +tools = Toolbox + +def quickstart(root: Optional[Application] = ..., script_name: str = ..., config: Optional[Union[TextIO, Dict[str, Any]]] = ...) -> None: ... diff --git a/typeshed/cherrypy/_cpreqbody.pyi b/typeshed/cherrypy/_cpreqbody.pyi new file mode 100644 index 0000000..5612f89 --- /dev/null +++ b/typeshed/cherrypy/_cpreqbody.pyi @@ -0,0 +1,9 @@ +from typing import BinaryIO, Optional +from cherrypy.lib.httputil import HeaderElement + +class Entity: + content_type: Optional[HeaderElement] = None + filename: Optional[str] = None + +class Part(Entity): + file: Optional[BinaryIO] = None diff --git a/typeshed/cherrypy/_cptree.pyi b/typeshed/cherrypy/_cptree.pyi new file mode 100644 index 0000000..24c9086 --- /dev/null +++ b/typeshed/cherrypy/_cptree.pyi @@ -0,0 +1,5 @@ +from typing import Any, Dict, Optional + +class Tree: + def __init__(self): ... + def mount(self, root: object, script_name: str = '', config: Optional[Dict[str, Dict[str, Any]]] = None) -> None: ... diff --git a/typeshed/cherrypy/daemon.pyi b/typeshed/cherrypy/daemon.pyi new file mode 100644 index 0000000..a5e3880 --- /dev/null +++ b/typeshed/cherrypy/daemon.pyi @@ -0,0 +1,7 @@ +from typing import Any, Dict, Optional, Sequence + +def start(configfiles: Optional[Sequence[Dict[str, Any]]] = None, + daemonize: bool = False, environment: Optional[Dict[str, str]] = None, + fastcgi: bool = False, scgi: bool = False, + pidfile: Optional[str] = None, imports: Optional[Sequence[str]] = None, + cgi: bool = False) -> None: ... diff --git a/typeshed/cherrypy/dispatch.pyi b/typeshed/cherrypy/dispatch.pyi new file mode 100644 index 0000000..84e05ab --- /dev/null +++ b/typeshed/cherrypy/dispatch.pyi @@ -0,0 +1,3 @@ +class Dispatcher: + def __init__(self): ... + def __call__(self, path_info: str) -> None: ... diff --git a/typeshed/cherrypy/lib/__init__.pyi b/typeshed/cherrypy/lib/__init__.pyi new file mode 100644 index 0000000..90a4cca --- /dev/null +++ b/typeshed/cherrypy/lib/__init__.pyi @@ -0,0 +1 @@ +from . import auth_digest, cptools, sessions diff --git a/typeshed/cherrypy/lib/auth_digest.pyi b/typeshed/cherrypy/lib/auth_digest.pyi new file mode 100644 index 0000000..f3a982d --- /dev/null +++ b/typeshed/cherrypy/lib/auth_digest.pyi @@ -0,0 +1,3 @@ +from typing import Callable, Dict + +def get_ha1_dict_plain(auth: Dict[str, str]) -> Callable[[str, str], str]: ... diff --git a/typeshed/cherrypy/lib/cptools.pyi b/typeshed/cherrypy/lib/cptools.pyi new file mode 100644 index 0000000..9e5d682 --- /dev/null +++ b/typeshed/cherrypy/lib/cptools.pyi @@ -0,0 +1 @@ +def validate_etags(autotags: bool = False, debug: bool = False) -> None: ... diff --git a/typeshed/cherrypy/lib/httputil.py b/typeshed/cherrypy/lib/httputil.py new file mode 100644 index 0000000..258268a --- /dev/null +++ b/typeshed/cherrypy/lib/httputil.py @@ -0,0 +1,5 @@ +from typing import Dict, Optional + +class HeaderElement: + value: Optional[str] + params: Dict[str, str] diff --git a/typeshed/cherrypy/lib/sessions.pyi b/typeshed/cherrypy/lib/sessions.pyi new file mode 100644 index 0000000..7cf2b91 --- /dev/null +++ b/typeshed/cherrypy/lib/sessions.pyi @@ -0,0 +1 @@ +def expire() -> None: ... diff --git a/typeshed/cherrypy/process/__init__.pyi b/typeshed/cherrypy/process/__init__.pyi new file mode 100644 index 0000000..547abce --- /dev/null +++ b/typeshed/cherrypy/process/__init__.pyi @@ -0,0 +1,3 @@ +from .wspbus import Bus + +bus: Bus = ... diff --git a/typeshed/cherrypy/process/wspbus.pyi b/typeshed/cherrypy/process/wspbus.pyi new file mode 100644 index 0000000..e035a4b --- /dev/null +++ b/typeshed/cherrypy/process/wspbus.pyi @@ -0,0 +1,6 @@ +from typing import Callable, Optional + +class Bus: + def __init__(self): ... + def subscribe(self, channel: str, callback: Optional[Callable] = None, priority: Optional[int] = None) -> None: ... + def publish(self, channel: str, *args: str, **kwargs: str) -> None: ... diff --git a/typeshed/cherrypy/test/__init__.pyi b/typeshed/cherrypy/test/__init__.pyi new file mode 100644 index 0000000..d533863 --- /dev/null +++ b/typeshed/cherrypy/test/__init__.pyi @@ -0,0 +1 @@ +from . import helper diff --git a/typeshed/cherrypy/test/helper.pyi b/typeshed/cherrypy/test/helper.pyi new file mode 100644 index 0000000..f13f253 --- /dev/null +++ b/typeshed/cherrypy/test/helper.pyi @@ -0,0 +1,14 @@ +from typing import List, Optional, Tuple, Type, Union +import unittest + +_Headers = List[Tuple[str, str]] +_InBody = Union[str, bytes] + +class CPWebCase(unittest.TestCase): + def getPage(self, url: str, headers: Optional[_Headers] = None, method: str = 'GET', body: Optional[_InBody] = None, protocol: Optional[str] = None, raise_subcls: Tuple[Type[Exception], ...] = ()) -> Tuple[str, _Headers, bytes]: ... + def assertStatus(self, status: str, msg: Optional[str] = None) -> None: ... + def assertHeader(self, key: str, value: Optional[str] = None, msg: Optional[str] = None) -> str: ... + def assertBody(self, value: _InBody, msg: Optional[str] = None) -> None: ... + def assertInBody(self, value: _InBody, msg: Optional[str] = None) -> None: ... + def assertNotInBody(self, value: _InBody, msg: Optional[str] = None) -> None: ... + body: str = ... diff --git a/typeshed/gpg/__init__.pyi b/typeshed/gpg/__init__.pyi new file mode 100644 index 0000000..e1d6d43 --- /dev/null +++ b/typeshed/gpg/__init__.pyi @@ -0,0 +1,5 @@ +from . import core +from . import errors +from .core import Data + +__all__ = ["Data", "core", "errors"] diff --git a/typeshed/gpg/core.pyi b/typeshed/gpg/core.pyi new file mode 100644 index 0000000..518c56a --- /dev/null +++ b/typeshed/gpg/core.pyi @@ -0,0 +1,30 @@ +from types import TracebackType +from typing import Callable, IO, Optional, Tuple, Type, TypeVar, Union +from . import gpgme as gpgme + +_Hook = TypeVar('_Hook', bound=Optional[Callable]) +_ReadCB = Callable[[int, _Hook], bytes] +_WriteCB = Callable[[bytes, _Hook], int] +_SeekCB = Callable[[int, int, _Hook], int] +_ReleaseCB = Callable[[_Hook], None] + +class GpgmeWrapper: + def __getattr__(self, key: str) -> Optional[Union[bool, Callable]]: ... + +class Data(GpgmeWrapper): + def __init__(self, + string: Optional[str] = None, + file: Optional[Union[str, IO]] = None, + offset: Optional[int] = None, + length: Optional[int] = None, + cbs: Optional[ + Union[Tuple[_ReadCB, _WriteCB, _SeekCB, _ReleaseCB], + Tuple[_ReadCB, _WriteCB, _SeekCB, _ReleaseCB, _Hook]] + ] = None, + copy: bool = True): ... + def __enter__(self) -> 'Data': ... + def __exit__(self, type: Optional[Type[BaseException]], + value: Optional[BaseException], + tb: Optional[TracebackType]) -> None: ... + def write(self, buffer: Union[str, bytes]) -> int: ... + def read(self, size: int = -1) -> Union[str, bytes]: ... diff --git a/typeshed/gpg/errors.pyi b/typeshed/gpg/errors.pyi new file mode 100644 index 0000000..65a9cfa --- /dev/null +++ b/typeshed/gpg/errors.pyi @@ -0,0 +1,2 @@ +class GpgError(Exception): + pass diff --git a/typeshed/gpg/gpgme.pyi b/typeshed/gpg/gpgme.pyi new file mode 100644 index 0000000..bfbd7f2 --- /dev/null +++ b/typeshed/gpg/gpgme.pyi @@ -0,0 +1,16 @@ +from typing import List + +class _gpgme_user_id: + name: str = ... + +class _gpgme_key: + fpr: str = ... + uids: List[_gpgme_user_id] = ... + +class _gpgme_import_status: + fpr: str = ... + +class _gpgme_op_import_result: + considered: int = ... + imported: int = ... + imports: List[_gpgme_import_status] = ... diff --git a/typeshed/gpg_exchange/__init__.pyi b/typeshed/gpg_exchange/__init__.pyi new file mode 100644 index 0000000..3013736 --- /dev/null +++ b/typeshed/gpg_exchange/__init__.pyi @@ -0,0 +1,3 @@ +from .exchange import Exchange + +__all__ = ["Exchange"] diff --git a/typeshed/gpg_exchange/exchange.pyi b/typeshed/gpg_exchange/exchange.pyi new file mode 100644 index 0000000..c9ee35c --- /dev/null +++ b/typeshed/gpg_exchange/exchange.pyi @@ -0,0 +1,47 @@ +from types import TracebackType +from typing import Any, Callable, IO, Optional, Sequence, Tuple, Type, Union +import gpg + +Passphrase = Union[str, Callable[[str, str, int, Optional[Any]], str]] + +class Exchange: + def __init__(self, armor: bool = True, home_dir: Optional[str] = None, + engine_path: Optional[str] = None, + passphrase: Optional[Passphrase] = None): ... + def __enter__(self) -> 'Exchange': ... + def __exit__(self, exc_type: Optional[Type[BaseException]], + exc_value: Optional[BaseException], + exc_tb: Optional[TracebackType]) -> None: ... + def generate_key(self, name: str, email: str, + comment: str = 'exchange generated key', + passphrase: Optional[str] = None) -> gpg.core.gpgme._gpgme_key: ... + def find_key(self, pattern: str) -> gpg.core.gpgme._gpgme_key: ... + def delete_key(self, pattern: str, secret: bool = False) -> gpg.core.gpgme._gpgme_key: ... + def _get_imported_key(self, import_result: gpg.core.gpgme._gpgme_op_import_result) -> gpg.core.gpgme._gpgme_key: ... + def import_key(self, pubkey: Union[str, bytes]) -> Tuple[gpg.core.gpgme._gpgme_key, gpg.core.gpgme._gpgme_op_import_result]: ... + def export_key(self, pattern: str) -> Union[str, bytes]: ... + @staticmethod + def _read_data(data: gpg.Data) -> Union[str, bytes]: ... + def _encrypt(self, plaintext: gpg.Data, ciphertext: gpg.Data, + recipients: Optional[Union[gpg.core.gpgme._gpgme_key, + Sequence[gpg.core.gpgme._gpgme_key]]], + passphrase: Optional[Passphrase], always_trust: bool) -> None: ... + def encrypt_text(self, data: Union[str, bytes], + recipients: Optional[Union[gpg.core.gpgme._gpgme_key, + Sequence[gpg.core.gpgme._gpgme_key]]] = None, + passphrase: Optional[Passphrase] = None, + always_trust: bool = False) -> Union[str, bytes]: ... + def encrypt_file(self, input_file: IO, output_file: IO, + recipients: Optional[Union[gpg.core.gpgme._gpgme_key, + Sequence[gpg.core.gpgme._gpgme_key]]] = None, + passphrase: Optional[Passphrase] = None, + always_trust: bool = False, + armor: Optional[bool] = None) -> Union[str, bytes]: ... + def _decrypt(self, ciphertext: gpg.Data, plaintext: gpg.Data, + passphrase: Optional[Passphrase] = None, + verify: bool = True) -> None: ... + def decrypt_text(self, data: bytes, passphrase: Optional[Passphrase] = None, + verify: bool = True) -> Union[str, bytes]: ... + def decrypt_file(self, input_file: IO, output_file: IO, + passphrase: Optional[Passphrase] = None, + verify: bool = True, armor: Optional[bool] = None) -> None: ... diff --git a/typeshed/keyring/__init__.pyi b/typeshed/keyring/__init__.pyi new file mode 100644 index 0000000..25b7850 --- /dev/null +++ b/typeshed/keyring/__init__.pyi @@ -0,0 +1,3 @@ +def get_password(keyring: str, username: str) -> str: ... +def set_password(keyring: str, username: str, password: str) -> None: ... +def delete_password(keyring: str, username: str) -> None: ... diff --git a/upload-session.sh b/upload-session.sh index 52809ae..a3815e5 100755 --- a/upload-session.sh +++ b/upload-session.sh @@ -4,7 +4,7 @@ # # Copyright 2017-2020 ICTU # Copyright 2017-2022 Leiden University -# Copyright 2017-2023 Leon Helwerda +# Copyright 2017-2024 Leon Helwerda # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,4 +18,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -/usr/bin/dbus-run-session -- bash -ce "eval \$(cat | gnome-keyring-daemon --unlock); /usr/local/bin/virtualenv.sh /usr/local/envs/controller /usr/local/bin/upload.py --scgi --debug --listen 127.0.0.1 --port 8143 --pidfile /var/log/upload/upload.pid --log-path /var/log/upload --upload-path /home/upload/upload --loopback" +if [ -t 0 ]; then + echo "Usage: | ./upload-session.sh" >&2 + echo "Start the upload service in a keyring and virtualenv environment" >&2 + exit 1 +fi + +/usr/bin/dbus-run-session -- bash -ce "eval \$(cat | gnome-keyring-daemon --unlock); /usr/local/bin/virtualenv.sh /usr/local/envs/controller gros-upload server --scgi --debug --listen 127.0.0.1 --port 8143 --pidfile /var/log/upload/upload.pid --log-path /var/log/upload --upload-path /home/upload/upload --loopback" diff --git a/upload.py b/upload.py deleted file mode 100644 index 7a0ee24..0000000 --- a/upload.py +++ /dev/null @@ -1,360 +0,0 @@ -""" -Listener server which accepts uploaded PGP-encrypted files. - -Copyright 2017-2020 ICTU -Copyright 2017-2022 Leiden University -Copyright 2017-2023 Leon Helwerda - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" - -import argparse -import configparser -import datetime -from hashlib import md5 -import json -from pathlib import Path -import shutil -from subprocess import Popen -import tempfile -import cherrypy -import cherrypy.daemon -import gpg -from gpg_exchange import Exchange -import keyring - -class Upload: - """ - Upload listener. - """ - - VERSION = "0.0.3" - - PGP_ARMOR_MIME = "application/pgp-encrypted" - PGP_BINARY_MIME = "application/x-pgp-encrypted-binary" - PGP_ENCRYPT_SUFFIX = ".gpg" - - def __init__(self, args, config): - self.args = args - self.config = config - - if self.args.keyring and self.args.loopback: - passphrase = self._get_passphrase - else: - passphrase = None - - self._gpg = Exchange(engine_path=self.args.engine, passphrase=passphrase) - - def _get_passphrase(self, hint, desc, prev_bad, hook=None): - # pylint: disable=unused-argument - return keyring.get_password(f'{self.args.keyring}-secret', 'privkey') - - @cherrypy.expose - @cherrypy.tools.json_in() - @cherrypy.tools.json_out() - def exchange(self): - """ - Exchange public keys. - """ - - data = cherrypy.request.json - if not isinstance(data, dict): - raise ValueError('Must provide a JSON object') - if 'pubkey' not in data: - raise ValueError('Must provide a pubkey') - pubkey = str(data['pubkey']) - - temp_dir = tempfile.mkdtemp() - with Exchange(home_dir=temp_dir, engine_path=self.args.engine) as import_gpg: - try: - key = import_gpg.import_key(pubkey)[0] - login = cherrypy.request.login - if login != '' and self.config.has_option('client', login): - if key.uids[0].name != self.config.get('client', login): - raise ValueError('Public key must match client') - - if key.uids[0].name not in self.args.accepted_keys: - raise ValueError('Must be an acceptable public key') - finally: - # Clean up temporary directory - shutil.rmtree(temp_dir) - - # Actual import - client_key = self._gpg.import_key(pubkey)[0] - - # Retrieve our own GPG key and encrypt it with the client key so that - # it cannot be intercepted by others (and thus others cannot send - # encrypted files in name of the client). - server_key = self._gpg.export_key(str(self.args.key)) - ciphertext = self._gpg.encrypt_text(server_key, client_key, - always_trust=True) - - return { - 'pubkey': str(ciphertext.decode('utf-8')) - } - - def _upload_gpg_file(self, input_file, path, binary=None, passphrase=None): - try: - with open(path, 'wb') as output_file: - self._gpg.decrypt_file(input_file, output_file, - armor=binary, passphrase=passphrase) - except (gpg.errors.GpgError, ValueError) as error: - # Write the (possibly encrypted) data to a separate file - with open(f"{path}.enc", 'wb') as output_file: - input_file.seek(0) - buf = True - while buf: - buf = input_file.read(1024) - if buf: - output_file.write(buf) - - raise ValueError(f'Decryption failed: {error}') from error - - @cherrypy.expose - @cherrypy.tools.json_out() - def upload(self, files): - """ - Perform an upload and import of GPG-encrypted files from a client - which has performed a key exchange. - """ - - if not isinstance(files, list): - files = [files] - - login = cherrypy.request.login - date = datetime.datetime.now().strftime('%Y-%m-%d') - directory = Path(self.args.upload_path) / login / date - if not directory.exists(): - directory.mkdir(mode=0o770, parents=True) - - for index, upload_file in enumerate(files): - name = upload_file.filename.split('/')[-1] - passphrase = None - binary = None - - if name == '': - raise ValueError(f'No name provided for file #{index}') - if name.endswith(self.PGP_ENCRYPT_SUFFIX): - name = name[:-len(self.PGP_ENCRYPT_SUFFIX)] - if self.args.keyring: - passphrase = \ - keyring.get_password(f'{self.args.keyring}-symmetric', - login) - else: - passphrase = self.config['symm'][login] - if name not in self.args.accepted_files: - raise ValueError(f'File #{index}: name {name} is unacceptable') - - if upload_file.content_type.value == self.PGP_ARMOR_MIME: - binary = False - elif upload_file.content_type.value == self.PGP_BINARY_MIME: - binary = True - - try: - self._upload_gpg_file(upload_file.file, directory / name, - binary=binary, passphrase=passphrase) - except ValueError as error: - raise ValueError(f'File {name}: {error}') from error - if name == self.args.import_dump: - process_args = [ - '/bin/bash', self.args.import_script, login, date, - self.args.database - ] - path = Path(self.args.import_path) / 'Scripts' - with Popen(process_args, stdout=None, stderr=None, cwd=path): - pass - - return { - 'success': True - } - - @classmethod - def json_error(cls, status, message, traceback, version): - """ - Handle HTTP errors by formatting the exception details as JSON. - """ - - cherrypy.response.headers['Content-Type'] = 'application/json' - return json.dumps({ - 'success': False, - 'error': { - 'status': status, - 'message': message, - 'traceback': traceback if cherrypy.request.show_tracebacks else None - }, - 'version': { - 'upload': cls.VERSION, - 'cherrypy': version - } if cherrypy.request.show_tracebacks else None - }) - -def get_ha1_keyring(name): - """ - Retrieve a function that provides an encoded variable containing the - username, realm and password for digest authentication. The `name` is - the keyring collection name. - """ - - def get_ha1(_, username): - """ - Retrieve the HA1 variable for a username from the keyring. - """ - - return str(keyring.get_password(name, username)) - - return get_ha1 - -def md5_hex(nonce): - """ - Encode as MD5. - """ - - return md5(nonce.encode('ISO-8859-1')).hexdigest() - -def ha1_nonce(username, realm, password): - """ - Create an encoded variant for the user's password in the realm. - """ - - return md5_hex(f'{username}:{realm}:{password}') - -def parse_args(config): - """ - Parse command line arguments. - """ - - work_dir = Path.cwd() - parser = argparse.ArgumentParser(description='Run upload listener') - parser.add_argument('--debug', action='store_true', default=False, - help='Output traces on web') - parser.add_argument('--listen', default=None, - help='Bind address (default: 0.0.0.0, 127.0.0.1 in debug') - parser.add_argument('--port', default=9090, type=int, - help='Port to listen to (default: 9090') - parser.add_argument('--log-path', dest='log_path', default=str(work_dir), - help='Path to store logs at in production') - parser.add_argument('--daemonize', action='store_true', default=False, - help='Run the server as a daemon') - parser.add_argument('--pidfile', help='Store process ID in file') - - parser.add_argument('--engine', default=config['server']['engine'], - help='GPG engine path') - parser.add_argument('--upload-path', dest='upload_path', - default=str(work_dir / 'upload'), - help='Upload path') - parser.add_argument('--accepted-files', dest='accepted_files', nargs='*', - default=config['server']['files'].split(' '), - type=set, help='List of filenames allowed for upload') - parser.add_argument('--database', default=config['import']['database'], - help='Database host to import dumps into') - parser.add_argument('--import-dump', default=config['import']['dump'], - dest='import_dump', help='File to import to a database') - parser.add_argument('--import-path', default=config['import']['path'], - dest='import_path', help='Path to the MonetDB importer') - parser.add_argument('--import-script', default=config['import']['script'], - dest='import_script', help='Path to the import script') - parser.add_argument('--key', default=config['server']['key'], - help='Fingerprint of server key pair') - parser.add_argument('--keyring', default=config['server']['keyring'], - help='Name of keyring containing authentication') - parser.add_argument('--realm', default=config['server']['realm'], - help='Name of Digest authentication realm') - parser.add_argument('--accepted-keys', dest='accepted_keys', nargs='*', - default=set(config['client'].values()), - type=set, help='List of accepted names for public keys') - parser.add_argument('--loopback', action='store_true', - help='Use loopback pinhole to read passphrase from keyring') - - server = parser.add_mutually_exclusive_group() - server.add_argument('--fastcgi', action='store_true', default=False, - help='Start a FastCGI server instead of HTTP') - server.add_argument('--scgi', action='store_true', default=False, - help='Start a SCGI server instead of HTTP') - server.add_argument('--cgi', action='store_true', default=False, - help='Start a CGI server instead of HTTP') - - return parser.parse_args() - -def main(): - """ - Main entry point. - """ - - config = configparser.RawConfigParser() - config.read('upload.cfg') - args = parse_args(config) - if args.listen is not None: - bind_address = args.listen - elif args.debug: - bind_address = '127.0.0.1' - else: - bind_address = '0.0.0.0' - - auth_key = config['server'].get('secret', '') - auth = dict((str(key), str(value)) for key, value in config['auth'].items()) - symm = dict((str(key), str(value)) for key, value in config['symm'].items()) - if args.keyring: - auth_keyring = keyring.get_password(f'{args.keyring}-secret', 'server') - if auth_keyring is not None: - auth_key = auth_keyring - elif auth_key != '': - keyring.set_password(f'{args.keyring}-secret', 'server', auth_key) - else: - raise ValueError('No server secret auth key provided') - - for user, password in auth.items(): - keyring.set_password(args.keyring, user, - ha1_nonce(user, args.realm, password)) - for user, passphrase in symm.items(): - keyring.set_password(f'{args.keyring}-symmetric', user, passphrase) - - ha1 = get_ha1_keyring(args.keyring) - else: - ha1 = cherrypy.lib.auth_digest.get_ha1_dict_plain(auth) - - if args.debug: - server = f'Cherrypy/{cherrypy.__version__}' - else: - server = 'Cherrypy' - - conf = { - 'global': { - }, - '/': { - 'error_page.default': Upload.json_error, - 'response.headers.server': server, - 'tools.auth_digest.on': True, - 'tools.auth_digest.realm': str(args.realm), - 'tools.auth_digest.get_ha1': ha1, - 'tools.auth_digest.key': str(auth_key) - } - } - log_path = Path(args.log_path) - cherrypy.config.update({ - 'server.max_request_body_size': 1000 * 1024 * 1024, - 'server.socket_host': bind_address, - 'server.socket_port': args.port, - 'request.show_tracebacks': args.debug, - 'log.screen': args.debug, - 'log.access_file': '' if args.debug else str(log_path / 'access.log'), - 'log.error_file': '' if args.debug else str(log_path / 'error.log'), - }) - - # Start the application and server daemon. - cherrypy.tree.mount(Upload(args, config), '/upload', conf) - cherrypy.daemon.start(daemonize=args.daemonize, pidfile=args.pidfile, - fastcgi=args.fastcgi, scgi=args.scgi, cgi=args.cgi) - - -if __name__ == '__main__': - main()