diff --git a/.flake8 b/.flake8 index 8e4ef5f..965498b 100644 --- a/.flake8 +++ b/.flake8 @@ -1,11 +1,11 @@ # Run flake8 (pycodestyle + pyflakes) check. # https://pycodestyle.readthedocs.io/en/latest/intro.html#error-codes # Ignored errors: -# - E501: line too long # - E265: block comment should start with '# ' (makes it easier to enable/disable code) # - W503: line break before binary operator (deprecated rule) # - W505: doc line too long [flake8] -ignore = E501,E265,W503,W505 -exclude = .git/,.virtualenv/,__pycache__/,build/,dist/ +ignore = E265,W503,W505 +max-line-length = 120 +exclude = .git/,.virtualenv/,.eggs/,__pycache__/,build/,dist/,submodules/ diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml index 1e9b66d..bc44e23 100644 --- a/.github/workflows/python-ci.yml +++ b/.github/workflows/python-ci.yml @@ -26,13 +26,12 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - python3 -m pip install --upgrade pip - python3 -m pip install flake8 - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + pip install -e '.[dev]' python3 setup.py install - - name: Lint code with Python ${{ matrix.python-version }} + - name: Check code with Python ${{ matrix.python-version }} run: | - make lint + make lint_local + make deadcode_local - name: Test code with Python ${{ matrix.python-version }} run: | - make test + make test_local diff --git a/Makefile b/Makefile index 2d604a3..d9ccb43 100644 --- a/Makefile +++ b/Makefile @@ -1,12 +1,21 @@ -PYFILES = mm_client/ examples/ - -all: lint: - flake8 ${PYFILES} + docker run -v ${CURDIR}:/apps registry.ubicast.net/docker/flake8:latest make lint_local + +lint_local: + flake8 . + +deadcode: + docker run -v ${CURDIR}:/apps registry.ubicast.net/docker/vulture:latest make deadcode_local + +deadcode_local: + vulture --exclude .eggs --min-confidence 90 . test: - python3 -m unittest discover tests/ -v + docker run -v ${CURDIR}:/apps registry.ubicast.net/docker/pytest:latest make test_local + +test_local: + pytest tests/ -vv --color=yes --log-level=DEBUG --cov=mm_client ${PYTEST_ARGS} build: clean python setup.py sdist bdist_wheel diff --git a/examples/fake_mediacoder.py b/examples/fake_mediacoder.py index 10875b5..1c348dd 100644 --- a/examples/fake_mediacoder.py +++ b/examples/fake_mediacoder.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- ''' Fake MediaCoder client for tests. ''' @@ -17,18 +16,39 @@ class FakeMediaCoder(MirisManagerClient): DEFAULT_CONF = { 'CAPABILITIES': ['record', 'network_record', 'web_control', 'screenshot'], } - PROFILES = {'main': {'has_password': False, 'can_live': False, 'name': 'main', 'label': 'Main', 'type': 'recorder'}} + PROFILES = { + 'main': { + 'has_password': False, + 'can_live': False, + 'name': 'main', + 'label': 'Main', + 'type': 'recorder' + } + } def handle_action(self, action, params): if action == 'START_RECORDING': logger.info('Starting recording with params %s', params) - self.set_status(status='initializing', status_message='', remaining_space='auto') + self.set_status( + status='initializing', + status_message='', + remaining_space='auto' + ) time.sleep(3) - self.set_status(status='running', status_message='', status_info='{"playlist": "/videos/BigBuckBunny_320x180.m3u8"}', remaining_space='auto') + self.set_status( + status='running', + status_message='', + status_info='{"playlist": "/videos/BigBuckBunny_320x180.m3u8"}', + remaining_space='auto' + ) elif action == 'STOP_RECORDING': logger.info('Stopping recording.') - self.set_status(status='ready', status_message='', remaining_space='auto') + self.set_status( + status='ready', + status_message='', + remaining_space='auto' + ) elif action == 'LIST_PROFILES': logger.info('Returning list of profiles.') @@ -36,18 +56,25 @@ def handle_action(self, action, params): elif action == 'GET_SCREENSHOT': self.set_status(remaining_space='auto') # Send remaining space to Miris Manager - self.set_screenshot('/var/lib/AccountsService/icons/%s' % (os.environ.get('USER') or 'root'), file_name='screen.png') + self.set_screenshot( + path='/var/lib/AccountsService/icons/%s' % (os.environ.get('USER') or 'root'), + file_name='screen.png' + ) logger.info('Screenshot sent.') else: - raise Exception('Unsupported action: %s.' % action) + raise NotImplementedError('Unsupported action: %s.' % action) if __name__ == '__main__': local_conf = sys.argv[1] if len(sys.argv) > 1 else None client = FakeMediaCoder(local_conf) client.update_capabilities() - client.set_status(status='ready', status_message='Ready to record', remaining_space='auto') + client.set_status( + status='ready', + status_message='Ready to record', + remaining_space='auto' + ) try: client.long_polling_loop() except KeyboardInterrupt: diff --git a/examples/miris_token.py b/examples/miris_token.py index 05489bb..3de38c5 100644 --- a/examples/miris_token.py +++ b/examples/miris_token.py @@ -18,24 +18,26 @@ } # GENERATE ONE TIME TOKEN +# Data will be passed to the recorder and also prevents another user +# from accessing the system if a recording is already in progress url = MM_URL + '/api/v3/users/create-token/' data = { 'purpose': 'control', 'system': SYSTEM, - 'data': json.dumps(user_info), # will be passed to the recorder and also prevents another user from accessing the system if a recording is already in progress + 'data': json.dumps(user_info), } r = requests.post(url, headers=headers, data=data).json() token = r['token'] #{'token': 'c8pse0v0gv312eg07m3vb29u6c78fcrlg5c1roo1', 'expires': '2022-01-14 02:56:13'} -# GENERATE FULL URL THE USER SHOULD BE 302ed to +# GENERATE FULL URL THE USER SHOULD BE REDIRECTED TO params = { 'profile': 'myprofile', 'title': 'my title', 'location': 'Room A', 'live_title': 'my live title', 'channel': 'mscspeaker', - 'logout_url': 'http://www.ubicast.eu', # you should probably redirect to the custom login page + 'logout_url': 'https://example.com', # you should probably redirect to the custom login page 'token': token, } diff --git a/examples/netcapture.py b/examples/netcapture.py index 1e9d187..a826576 100644 --- a/examples/netcapture.py +++ b/examples/netcapture.py @@ -10,9 +10,48 @@ def get_status(session): - status_dict = s.get(URL + '/api/v3/fleet/systems/get-status/', headers=headers, params={'profile': PROFILE}).json() + status_dict = session.get( + URL + '/api/v3/fleet/systems/get-status/', + headers=headers, + params={'profile': PROFILE} + ).json() ''' - status dict: {'last_status_update': '20200227112959', 'last_status_update_display': '2020-02-27 11:29:59', 'last_connection': '2020-02-27 11:29:57', 'profile': 'ndi', 'status': 'RUNNING', 'status_info': {'status_message': 'Recording in progress', 'audio': {'master': {'volume': 1.0, 'muted': False}}, 'video': [{'type': 'ndivsource', 'name': 'source-f41f', 'device': 'v4l-HDMI-1-pci-0000:01:00.0', 'capture': '1280x720@25', 'signal': 'fake'}], 'playlist': '/hls/944d/adaptive.m3u8', 'time_in_sec': 2, 'timecode': '0:00:02', 'record_folder': '/home/ubicast/mediacoder/media/20200227-112957-09bd'}, 'status_display': 'Recording in progress (profile: ndi)', 'remaining_space': 911891, 'remaining_time': 0, 'online': True, 'auto_refresh': False, 'screenshot_name': 'screenshots/ubi-box-e0d55ec52008_2020-02-27_10-30-00.jpg', 'screenshot_date': '2020-02-27 11:30:00', 'screenshot_outdated': False, 'messages_info': 20, 'messages_warning': 2, 'messages_error': 17, 'last_error_message': None, 'last_error_date': None} + status dict sample: + { + 'last_status_update': '20200227112959', + 'last_status_update_display': '2020-02-27 11:29:59', + 'last_connection': '2020-02-27 11:29:57', + 'profile': 'ndi', + 'status': 'RUNNING', + 'status_info': { + 'status_message': 'Recording in progress', + 'audio': {'master': {'volume': 1.0, 'muted': False}}, + 'video': [{ + 'type': 'ndivsource', + 'name': 'source-f41f', + 'device': 'v4l-HDMI-1-pci-0000:01:00.0', + 'capture': '1280x720@25', + 'signal': 'fake' + }], + 'playlist': '/hls/944d/adaptive.m3u8', + 'time_in_sec': 2, + 'timecode': '0:00:02', + 'record_folder': '/home/ubicast/mediacoder/media/20200227-112957-09bd' + }, + 'status_display': 'Recording in progress (profile: ndi)', + 'remaining_space': 911891, + 'remaining_time': 0, + 'online': True, + 'auto_refresh': False, + 'screenshot_name': 'screenshots/ubi-box-e0d55ec52008_2020-02-27_10-30-00.jpg', + 'screenshot_date': '2020-02-27 11:30:00', + 'screenshot_outdated': False, + 'messages_info': 20, + 'messages_warning': 2, + 'messages_error': 17, + 'last_error_message': None, + 'last_error_date': None + } status_dict["status"] can contain: "READY": idle @@ -23,18 +62,23 @@ def get_status(session): return status_dict.get('status') -with requests.Session() as s: - params = { +with requests.Session() as session: + data = { 'profile': PROFILE, 'async': 'no', 'action': 'START_RECORDING', 'speaker_email': 'user@domain.com', 'course_id': 'mscspeaker' } - # list of mediaserver supported parameters here: https://sandbox.ubicast.tv/static/mediaserver/docs/api/api.html#api-v2-medias-add + # list of mediaserver supported parameters here: + # https://ubicast.tv/static/mediaserver/docs/api/api.html#api-v2-medias-add print('Start recording') - response = s.post(URL + '/api/v3/fleet/control/run-command/', headers=headers, data=params).json() + response = session.post( + URL + '/api/v3/fleet/control/run-command/', + headers=headers, + data=data + ).json() #{'uid': '89b15583-7528-4c0c-a18e-28155be08d21', 'status': 'DONE', 'message': 'Recording started'} if response.get('error'): @@ -45,13 +89,17 @@ def get_status(session): print(response.get('message')) # can be UNAVAILABLE, READY, INITIALIZING, RUNNING # https://mirismanager.ubicast.eu/static/docs/api/values.html - while not get_status(s) == 'RUNNING': + while not get_status(session) == 'RUNNING': time.sleep(1) - if get_status(s) == 'UNAVAILABLE': + if get_status(session) == 'UNAVAILABLE': print('Profile does not exist or no system is online') sys.exit(1) print('System is recording, stopping now') print('Stop recording') - params['action'] = 'STOP_RECORDING' - s.post(URL + '/api/v3/fleet/control/run-command/', headers=headers, data=params) + data['action'] = 'STOP_RECORDING' + session.post( + URL + '/api/v3/fleet/control/run-command/', + headers=headers, + data=data + ) diff --git a/examples/ping_server.py b/examples/ping_server.py index 40bcbbb..9fb3016 100644 --- a/examples/ping_server.py +++ b/examples/ping_server.py @@ -1,14 +1,13 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- ''' Script to ping a Miris Manager server. ''' -import os import sys +from pathlib import Path if __name__ == '__main__': - sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + sys.path.append(str(Path(__file__).resolve().parent.parent)) from mm_client.client import MirisManagerClient local_conf = sys.argv[1] if len(sys.argv) > 1 else None diff --git a/examples/poll_profile.py b/examples/poll_profile.py index f1fe705..fb75c54 100644 --- a/examples/poll_profile.py +++ b/examples/poll_profile.py @@ -1,16 +1,15 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- ''' Script to poll a profile status in a Miris Manager server. The script is intended to be used with a user API key and not a system API key. ''' -import os import sys import time +from pathlib import Path if __name__ == '__main__': - sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + sys.path.append(str(Path(__file__).resolve().parent.parent)) from mm_client.client import MirisManagerClient local_conf = sys.argv[1] if len(sys.argv) > 1 else None diff --git a/examples/recorder_controller.py b/examples/recorder_controller.py index 47b196d..668f641 100644 --- a/examples/recorder_controller.py +++ b/examples/recorder_controller.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- ''' An example of Miris Manager client usage. This script is intended to control a recorder. diff --git a/examples/screen_controller.py b/examples/screen_controller.py index 342361a..ee58923 100644 --- a/examples/screen_controller.py +++ b/examples/screen_controller.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- ''' An example of Miris Manager client usage. This script is intended to send screenshot and handle click requests. @@ -28,7 +27,10 @@ def handle_action(self, action, params): elif action == 'GET_SCREENSHOT': self.set_status(remaining_space='auto') # Send remaining space to Miris Manager - self.set_screenshot('/var/lib/AccountsService/icons/%s' % (os.environ.get('USER') or 'root'), file_name='screen.png') + self.set_screenshot( + path='/var/lib/AccountsService/icons/%s' % (os.environ.get('USER') or 'root'), + file_name='screen.png' + ) logger.info('Screenshot sent.') elif action == 'SIMULATE_CLICK': diff --git a/examples/send_message.py b/examples/send_message.py index 9fd41e8..a2cb6ea 100644 --- a/examples/send_message.py +++ b/examples/send_message.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- ''' Script to send a message ''' diff --git a/examples/wol_relay.py b/examples/wol_relay.py index 3d20b5f..d843161 100644 --- a/examples/wol_relay.py +++ b/examples/wol_relay.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- ''' An example of Miris Manager client usage. This script is intended to create devices acting as wake on lan relay and video displayer. diff --git a/mm_client/__init__.py b/mm_client/__init__.py index 773d930..5fa6331 100644 --- a/mm_client/__init__.py +++ b/mm_client/__init__.py @@ -1 +1 @@ -__version__ = '5.5' +__version__ = '6.0' diff --git a/mm_client/client.py b/mm_client/client.py index cf9ce4f..b7ee6e4 100644 --- a/mm_client/client.py +++ b/mm_client/client.py @@ -1,9 +1,11 @@ ''' -Miris Manager client class +Miris Manager client main module ''' import logging -import os +from pathlib import Path + import requests + from .lib import configuration as configuration_lib from .lib import info as info_lib from .lib import long_polling as long_polling_lib @@ -70,13 +72,14 @@ def get_url_info(self, url_or_action): return {'url': url_or_action} if url_or_action not in self.conf['API_CALLS']: raise MirisManagerRequestError( - 'Invalid url requested: %s does not exist in API_CALLS configuration.' % url_or_action, + f'Invalid url requested: {url_or_action} does not exist in API_CALLS configuration.', status_code=0, error_code='invalid_url' ) return self.conf['API_CALLS'][url_or_action] - def _request(self, url, method='get', headers=None, params=None, data=None, files=None, anonymous=None, timeout=None): + def _request(self, url, method='get', headers=None, params=None, + data=None, files=None, anonymous=None, timeout=None): req = getattr(requests, method)( url=self.conf['SERVER_URL'] + url, headers=headers, @@ -133,7 +136,8 @@ def _register(self): logger.info('System registration done.') return True - def api_request(self, url_or_action, method='get', headers=None, params=None, data=None, files=None, anonymous=None, timeout=None): + def api_request(self, url_or_action, method='get', headers=None, params=None, + data=None, files=None, anonymous=None, timeout=None): self.check_conf() url_info = self.get_url_info(url_or_action) if anonymous is None: @@ -144,12 +148,13 @@ def api_request(self, url_or_action, method='get', headers=None, params=None, da # Register system if no API key and auto registration if not self.conf.get('API_KEY'): if not self.conf['AUTO_REGISTRATION']: - raise Exception('The client auto registration is disabled and no API_KEY is set in conf file, please set one or turn on auto registration.') + raise ValueError('The client auto registration is disabled and no API_KEY is set in conf file, ' + 'please set one or turn on auto registration.') try: self._register() except Exception as e: logger.error('Registration failed: %s', e) - raise Exception('Registration failed: %s' % e) + raise # Add signature in headers # headers with "_" are ignored by Django _headers = {'api-key': self.conf['API_KEY']} @@ -160,7 +165,15 @@ def api_request(self, url_or_action, method='get', headers=None, params=None, da if headers: _headers.update(headers) # Make API request - response = self._request(url_info['url'], method=url_info.get('method', method), headers=_headers, params=params, data=data, files=files, timeout=timeout) + response = self._request( + url_info['url'], + method=url_info.get('method', method), + headers=_headers, + params=params, + data=data, + files=files, + timeout=timeout + ) return response def long_polling_loop(self): @@ -203,7 +216,8 @@ def update_capabilities(self): response = self.api_request('SET_INFO', data=data) return response - def set_status(self, status=None, status_info=None, status_message=None, profile=None, remaining_space=None, remaining_time=None): + def set_status(self, status=None, status_info=None, status_message=None, + profile=None, remaining_space=None, remaining_time=None): data = {} if status is not None: data['status'] = status @@ -227,7 +241,7 @@ def set_status(self, status=None, status_info=None, status_message=None, profile def set_screenshot(self, path, file_name=None): with open(path, 'rb') as file_obj: response = self.api_request('SET_SCREENSHOT', files=dict( - screenshot=(file_name or os.path.basename(path), file_obj) + screenshot=(file_name or Path(path).name, file_obj) )) return response diff --git a/mm_client/lib/configuration.py b/mm_client/lib/configuration.py index aa1beaa..5cac3d2 100644 --- a/mm_client/lib/configuration.py +++ b/mm_client/lib/configuration.py @@ -4,8 +4,9 @@ ''' import json import logging -import os import re +from pathlib import Path + from ..conf import BASE_CONF logger = logging.getLogger('mm_client.lib.configuration') @@ -18,14 +19,15 @@ def load_conf(default_conf=None, local_conf=None): for index, conf_override in enumerate((default_conf, local_conf)): if not conf_override: continue + if isinstance(conf_override, str): + conf_override = Path(conf_override) if isinstance(conf_override, dict): for key, val in conf_override.items(): if not key.startswith('_'): conf[key] = val - elif isinstance(conf_override, str): - if os.path.exists(conf_override): - with open(conf_override, 'r') as fo: - content = fo.read() + elif isinstance(conf_override, Path): + if conf_override.exists(): + content = conf_override.read_text() content = re.sub(r'\n\s*//.*', '\n', content) # remove comments conf_mod = json.loads(content) if content else None if not conf_mod: @@ -33,7 +35,7 @@ def load_conf(default_conf=None, local_conf=None): else: logger.debug('Config file "%s" loaded.', conf_override) if not isinstance(conf_mod, dict): - raise ValueError('The configuration in "%s" is not a dict.' % conf_override) + raise ValueError(f'The configuration in "{conf_override}" is not a dict.') conf.update(conf_mod) else: logger.debug('Config file does not exists, using default config.') @@ -45,20 +47,25 @@ def load_conf(default_conf=None, local_conf=None): def update_conf(local_conf, key, value): - if not local_conf or not isinstance(local_conf, str): + if not local_conf: + logger.debug('Cannot update configuration, "local_conf" is not set.') + return False + if isinstance(local_conf, str): + local_conf = Path(local_conf) + elif not isinstance(local_conf, Path): logger.debug('Cannot update configuration, "local_conf" is not a path.') - return - content = '' - if os.path.isfile(local_conf): - with open(local_conf, 'r') as fo: - content = fo.read() - content = content.strip() - data = json.loads(content) if content else dict() + return False + + if local_conf.is_file(): + content = local_conf.read_text().strip() + else: + content = '' + data = json.loads(content) if content else {} data[key] = value new_content = json.dumps(data, sort_keys=True, indent=4) - with open(local_conf, 'w') as fo: - fo.write(new_content) + local_conf.write_text(new_content) logger.debug('Configuration file "%s" updated: "%s" set to "%s".', local_conf, key, value) + return True def check_conf(conf): diff --git a/mm_client/lib/long_polling.py b/mm_client/lib/long_polling.py index d3ab427..fc3e4e3 100644 --- a/mm_client/lib/long_polling.py +++ b/mm_client/lib/long_polling.py @@ -74,7 +74,7 @@ def call_long_polling(self): self.client.set_command_status(uid, 'FAILED', str(e)) if os.environ.get('CI_PIPELINE_ID'): # propagate exception so that it can be detected in CI - raise Exception(e) + raise else: self.client.set_command_status(uid, 'DONE', result) finally: @@ -88,10 +88,10 @@ def process_long_polling(self, response): if self.client.conf.get('API_KEY'): invalid = check_signature(self.client, response) if invalid: - raise Exception('Invalid signature: %s' % invalid) + raise ValueError('Invalid signature: %s' % invalid) action = response.get('action') if not action: - raise Exception('No action received.') + raise ValueError('No action received.') params = response.get('params', dict()) logger.debug('Received command "%s": %s.', response.get('uid'), action) if action == 'PING': diff --git a/mm_client/lib/signing.py b/mm_client/lib/signing.py index 51115c1..d9f2ed7 100644 --- a/mm_client/lib/signing.py +++ b/mm_client/lib/signing.py @@ -13,7 +13,7 @@ def get_signature(client): if not client.conf.get('SECRET_KEY') or not client.conf.get('API_KEY'): - return dict() + return {} utime = datetime.datetime.utcnow().strftime('%Y-%m-%d_%H-%M-%S_%f') to_sign = 'time=%s|api_key=%s' % (utime, client.conf['API_KEY']) hm = hmac.new( diff --git a/mm_client/lib/ssh_tunnel.py b/mm_client/lib/ssh_tunnel.py index a3d09af..3f052b0 100644 --- a/mm_client/lib/ssh_tunnel.py +++ b/mm_client/lib/ssh_tunnel.py @@ -1,7 +1,9 @@ ''' Miris Manager SSH tunnel management This module is not intended to be used directly, only the client class should be used. -The SSH tunnel goal is to access the system web interface (HTTPS) from Miris Manager using a connection from the system to the Miris Manager. + +The SSH tunnel goal is to access the system web interface (HTTPS) from +Miris Manager using a connection from the system to the Miris Manager. ''' import logging import os @@ -10,38 +12,52 @@ import multiprocessing import re import signal +from pathlib import Path logger = logging.getLogger('mm_client.lib.ssh_tunnel') +class MirisManagerTunnelError(Exception): + pass + + def get_ssh_public_key(): - ssh_key_path = os.path.join(os.path.expanduser('~/.ssh/miris-manager-client-key')) - ssh_dir = os.path.dirname(ssh_key_path) - if not os.path.exists(ssh_dir): - os.makedirs(ssh_dir) - os.chmod(ssh_dir, 0o700) - if os.path.exists(ssh_key_path): - if not os.path.exists(ssh_key_path + '.pub'): - raise Exception('Weird state detetected: "%s" exists but not "%s" !' % (ssh_key_path, ssh_key_path + '.pub')) + ssh_dir = Path('~/.ssh').expanduser() + ssh_key_path = ssh_dir / 'miris-manager-client-key' + ssh_pub_path = ssh_dir / 'miris-manager-client-key.pub' + if not ssh_dir.exists(): + ssh_dir.mkdir(parents=True) + ssh_dir.chmod(0o700) + if ssh_key_path.exists(): + if not ssh_pub_path.exists(): + raise MirisManagerTunnelError( + f'Weird state detetected: "{ssh_key_path}" exists but not "{ssh_pub_path}" !' + ) logger.debug('Using existing SSH key: "%s".', ssh_key_path) else: logger.info('Creating new SSH key: "%s".', ssh_key_path) - p = subprocess.Popen(['ssh-keygen', '-b', '4096', '-f', ssh_key_path, '-N', ''], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + p = subprocess.Popen( + ['ssh-keygen', '-b', '4096', '-f', str(ssh_key_path), '-N', ''], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT + ) p.communicate(input=b'\n\n\n') if p.returncode != 0: - out = p.stdout.decode('utf-8') + '\n' + p.stderr.decode('utf-8') - raise Exception('Failed to generate SSH key:\n%s' % out) - os.chmod(ssh_key_path, 0o600) - os.chmod(ssh_key_path + '.pub', 0o600) - with open(ssh_key_path + '.pub', 'r') as fo: - public_key = fo.read() + out = p.stdout.decode('utf-8').strip() + raise MirisManagerTunnelError( + f'Failed to generate SSH key:\n{out}' + ) + ssh_key_path.chmod(0o600) + ssh_pub_path.chmod(0o600) + public_key = ssh_pub_path.read_text() return public_key def prepare_ssh_command(host, info): - ssh_key_path = os.path.join(os.path.expanduser('~/.ssh/miris-manager-client-key')) + ssh_key_path = Path('~/.ssh/miris-manager-client-key').expanduser() command = ['ssh', - '-i', ssh_key_path, + '-i', str(ssh_key_path), '-nvNT', '-o', 'IdentitiesOnly=yes', '-o', 'NumberOfPasswordPrompts=0', @@ -61,17 +77,39 @@ def __init__(self, client, status_callback=None): self.client = client self.status_callback = status_callback self.pattern_list = [ - dict(id='connecting', pattern=re.compile(r'debug1: Connecting to (?P[^ ]+) \[(?P[0-9\.]{7,15})\] port (?P\d{1,5}).\r\n')), - dict(id='connected', pattern=re.compile(r'debug1: Connection established.\r\n')), - dict(id='authenticated', pattern=re.compile(r'debug1: Authentication succeeded \((?P[^\)]+)\).\r\n')), - dict(id='authenticated', pattern=re.compile(r'Authenticated to (?P[^ ]+) \(\[(?P[0-9\.]{7,15})\]:(?P\d{1,5})\).\r\n')), - dict(id='running', pattern=re.compile(r'debug1: Entering interactive session.\r\n')), - dict(id='not_known', pattern=re.compile(r'ssh: [^:]+: Name or service not known\r\n')), - dict(id='port_refused', pattern=re.compile(r'Warning: remote port forwarding failed for listen port (?P\d{1,5})')), - dict(id='refused', pattern=re.compile(r'ssh: connect to host [^:]+: Connection refused\r\n')), - dict(id='control_refused', pattern=re.compile(r'connect_to (?P[^ ]+) port (?P\d{1,5}): failed\.\r\n')), - dict(id='denied', pattern=re.compile(r'Permission denied \(publickey,password\).\r\n')), - dict(id='closed', pattern=re.compile(r'Connection to (?P[^ ]+) closed.\r\n')), + dict(id='connecting', pattern=re.compile( + r'debug1: Connecting to (?P[^ ]+) \[(?P[0-9\.]{7,15})\] port (?P\d{1,5}).\r\n' + )), + dict(id='connected', pattern=re.compile( + r'debug1: Connection established.\r\n' + )), + dict(id='authenticated', pattern=re.compile( + r'debug1: Authentication succeeded \((?P[^\)]+)\).\r\n' + )), + dict(id='authenticated', pattern=re.compile( + r'Authenticated to (?P[^ ]+) \(\[(?P[0-9\.]{7,15})\]:(?P\d{1,5})\).\r\n' + )), + dict(id='running', pattern=re.compile( + r'debug1: Entering interactive session.\r\n' + )), + dict(id='not_known', pattern=re.compile( + r'ssh: [^:]+: Name or service not known\r\n' + )), + dict(id='port_refused', pattern=re.compile( + r'Warning: remote port forwarding failed for listen port (?P\d{1,5})' + )), + dict(id='refused', pattern=re.compile( + r'ssh: connect to host [^:]+: Connection refused\r\n' + )), + dict(id='control_refused', pattern=re.compile( + r'connect_to (?P[^ ]+) port (?P\d{1,5}): failed\.\r\n' + )), + dict(id='denied', pattern=re.compile( + r'Permission denied \(publickey,password\).\r\n' + )), + dict(id='closed', pattern=re.compile( + r'Connection to (?P[^ ]+) closed.\r\n' + )), ] self.loop_ssh_tunnel = False self.process = None @@ -91,7 +129,7 @@ def __init__(self, client, status_callback=None): def establish_tunnel(self): public_key = None response = None - logger.debug('Establishing new tunnel to %s' % self.client.conf['SERVER_URL']) + logger.debug('Establishing new tunnel to %s', self.client.conf['SERVER_URL']) self._stop_reader() self._try_closing_process() self.update_ssh_state('state', 'prepare tunnel') @@ -104,7 +142,7 @@ def establish_tunnel(self): self.update_ssh_state('control_port', 0) self.update_ssh_state('maintenance_port', 0) self.update_ssh_state('command', ['PREPARE_TUNNEL', self.client.conf['SERVER_URL']]) - logger.error('Cannot prepare ssh tunnel : %s' % str(e)) + logger.error('Cannot prepare ssh tunnel : %s', str(e)) return ssh_user = response.get('ssh_user') if ssh_user and ssh_user != self.ssh_tunnel_state['ssh_user']: @@ -124,7 +162,12 @@ def establish_tunnel(self): self.update_ssh_state('command', cmd) logger.info('Starting SSH with command:\n %s', ' '.join(cmd)) if self.loop_ssh_tunnel: - self.process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, preexec_fn=os.setsid) + self.process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + preexec_fn=os.setsid + ) self.stdout_queue = multiprocessing.Queue() self.stdout_reader = AsynchronousFileReader(self.process.stdout, self.stdout_queue) self.stdout_reader.start() @@ -136,11 +179,11 @@ def establish_tunnel(self): def update_ssh_state(self, key, value): if key == 'state' and self.ssh_tunnel_state.get('state') != value: - logger.info('SSH state changed to %s' % value) + logger.info('SSH state changed to %s', value) if self.ssh_tunnel_state.get(key) is not None: self.ssh_tunnel_state[key] = value else: - logger.warning('Key %s not exists in ssh state dict' % key) + logger.warning('Key %s not exists in ssh state dict', key) if self.status_callback: self.status_callback(self.ssh_tunnel_state) @@ -240,7 +283,7 @@ def read_ssh_stdout(self): ssh_logs = str(e) self.update_ssh_state('state', 'error') self.update_ssh_state('last_tunnel_info', ssh_logs) - logger.error('SSH tunnel process has terminated with: %s' % ssh_logs) + logger.error('SSH tunnel process has terminated with: %s', ssh_logs) need_retry = True else: # process is still alive @@ -256,11 +299,21 @@ def read_ssh_stdout(self): break if not pattern_id_found: if ssh_stdout.startswith('debug1:') or ssh_stdout.startswith('OpenSSH_'): - logger.debug('[SSH stdout] %s' % ssh_stdout) + logger.debug('[SSH stdout] %s', ssh_stdout) else: - logger.warning('[SSH stdout] %s' % ssh_stdout) + logger.warning('[SSH stdout] %s', ssh_stdout) elif pattern_id_found not in ['connecting', 'connected', 'authenticated', 'running']: - logger.error('Need to retry tunnel (ssh port: {ssh_port}, remote control port: {control_port}, remote maintenance port: {maintenance_port}) because ssh command failed in stdout %s'.format(**self.ssh_tunnel_state) % pattern_id_found) + logger.error( + ( + 'Need to retry tunnel ' + '(ssh port: %s, remote control port: %s, remote maintenance port: %s) ' + 'because ssh command failed in stdout %s' + ), + self.ssh_tunnel_state.get('ssh_port'), + self.ssh_tunnel_state.get('control_port'), + self.ssh_tunnel_state.get('maintenance_port'), + pattern_id_found + ) need_retry = True break except OSError as e: diff --git a/setup.cfg b/setup.cfg index 1a64fcf..334d76f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -40,6 +40,9 @@ setup_requires = [options.extras_require] dev = flake8 + pytest + pytest-cov + vulture [bdist_wheel] universal = 1 diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..851cd8b --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,18 @@ +import sys +from pathlib import Path + +import urllib3 + +import pytest + + +@pytest.fixture(scope='session', autouse=True) +def disable_urllib3_warnings(): + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + + +@pytest.fixture(scope='session', autouse=True) +def add_client_to_path(): + path = Path(__file__).resolve().parent.parent + sys.path.pop(0) # Remove current dir + sys.path.insert(0, str(path)) diff --git a/tests/test_client.py b/tests/test_client.py index 337b13f..da0f135 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -1,15 +1,5 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -''' -Miris Manager client test file. -''' -from unittest.mock import patch import json -import logging -import os -import sys -import unittest -import urllib3 +from unittest.mock import patch CONFIG = { 'SERVER_URL': 'https://mmctest' @@ -32,33 +22,12 @@ def json(self): return MockResponse(None, 404) -class MMClientTest(unittest.TestCase): - maxDiff = None - - def setUp(self): - print('\n\033[96m----- %s.%s -----\033[0m' % (self.__class__.__name__, self._testMethodName)) - # Setup logging - logging.basicConfig( - level=logging.DEBUG, - format='%(asctime)s %(name)s %(levelname)s %(message)s', - stream=sys.stdout - ) - urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - # Setup sys path - sys.path.pop(0) # Remove current dir - src_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) - sys.path.insert(0, src_dir) - - @patch('requests.get', side_effect=mocked_requests_get) - def test_client(self, mock_get): - from mm_client.client import MirisManagerClient - mmc = MirisManagerClient(local_conf=CONFIG) - response = mmc.api_request('PING') - self.assertTrue(isinstance(response, dict)) - self.assertEqual(response['version'], '8.0.0') - - self.assertEqual(len(mock_get.call_args_list), 1) - +@patch('requests.get', side_effect=mocked_requests_get) +def test_client(mock_get): + from mm_client.client import MirisManagerClient + mmc = MirisManagerClient(local_conf=CONFIG) + response = mmc.api_request('PING') + assert isinstance(response, dict) + assert response['version'] == '8.0.0' -if __name__ == '__main__': - unittest.main() + assert len(mock_get.call_args_list) == 1 diff --git a/tests/test_conf.py b/tests/test_conf.py new file mode 100644 index 0000000..38b9edc --- /dev/null +++ b/tests/test_conf.py @@ -0,0 +1,75 @@ +from pathlib import Path + +import pytest + + +@pytest.fixture() +def conf_path(): + path = Path('/tmp/mm-conf.json') + path.write_text('{"SERVER_URL": "https://test"}') + yield path + path.unlink(missing_ok=True) + + +def test_conf_file__valid(conf_path): + from mm_client.lib.configuration import load_conf, update_conf + + conf = load_conf(default_conf=conf_path) + assert conf['SERVER_URL'] == 'https://test' + + updated = update_conf(conf_path, 'test', 'val') + assert updated is True + + +def test_conf_file__does_not_exist(conf_path): + from mm_client.lib.configuration import load_conf, update_conf + + conf_path.unlink() + conf = load_conf(default_conf=conf_path) + assert conf['SERVER_URL'] == 'https://mirismanager' + + updated = update_conf(conf_path, 'test', 'val') + assert updated is True + + +def test_conf_dict(): + from mm_client.lib.configuration import load_conf, update_conf + + conf = load_conf( + default_conf={'SERVER_URL': 'https://nope'}, + local_conf={'SERVER_URL': 'https://test'} + ) + assert conf['SERVER_URL'] == 'https://test' + + updated = update_conf({'SERVER_URL': 'https://test'}, 'test', 'val') + assert updated is False + + +def test_conf_default(): + from mm_client.lib.configuration import load_conf, update_conf + + conf = load_conf() + assert conf['SERVER_URL'] == 'https://mirismanager' + + updated = update_conf(None, 'test', 'val') + assert updated is False + + +@pytest.mark.parametrize('conf, is_valid', [ + pytest.param( + {'SERVER_URL': 'https://mirismanager'}, + False, + id='default'), + pytest.param( + {'SERVER_URL': 'https://test/'}, + True, + id='valid'), +]) +def test_conf_check(conf, is_valid): + from mm_client.lib.configuration import check_conf + + if is_valid: + check_conf(conf) + else: + with pytest.raises(ValueError): + check_conf(conf) diff --git a/tests/test_info.py b/tests/test_info.py new file mode 100644 index 0000000..a5a271e --- /dev/null +++ b/tests/test_info.py @@ -0,0 +1,25 @@ + + +def test_get_host_info(): + from mm_client.lib.info import get_host_info + + expected = ['hostname', 'local_ip', 'mac'] + + info = get_host_info(url='https://localhost') + assert sorted(info.keys()) == expected + for field in expected: + assert info[field] + + +def test_get_free_space_bytes(): + from mm_client.lib.info import get_free_space_bytes + + remaining = get_free_space_bytes('/home') + assert remaining > 0 + + +def test_get_remaining_space(): + from mm_client.lib.info import get_remaining_space + + remaining = get_remaining_space() + assert remaining > 0 diff --git a/tests/test_signature.py b/tests/test_signature.py new file mode 100644 index 0000000..8d1610f --- /dev/null +++ b/tests/test_signature.py @@ -0,0 +1,62 @@ +import datetime + +import pytest + + +def test_signature__default(): + from mm_client.client import MirisManagerClient + from mm_client.lib.signing import get_signature, check_signature + + client = MirisManagerClient() + + signature = get_signature(client) + assert signature == {} + + assert check_signature(client, {}) is None + + +def test_signature__configured(): + from mm_client.client import MirisManagerClient + from mm_client.lib.signing import get_signature, check_signature + + conf = { + 'SECRET_KEY': 'the secret key', + 'API_KEY': 'the API key', + } + client = MirisManagerClient(conf) + + signature = get_signature(client) + assert sorted(signature.keys()) == ['hmac', 'time'] + + assert check_signature(client, signature) is None + + +@pytest.mark.parametrize('signature, expected', [ + pytest.param( + {}, + 'some mandatory data are missing.', + id='missing'), + pytest.param( + {'time': 'invalid', 'hmac': 'test'}, + 'the received time is invalid.', + id='date'), + pytest.param( + {'time': '2000-01-01_00-00-00_000', 'hmac': 'test'}, + 'the difference between the request time and the current time is too large.', + id='expired'), + pytest.param( + {'time': datetime.datetime.utcnow().strftime('%Y-%m-%d_%H-%M-%S_%f'), 'hmac': 'test'}, + 'the received and computed HMAC values do not match.', + id='hmac'), +]) +def test_signature__invalid(signature, expected): + from mm_client.client import MirisManagerClient + from mm_client.lib.signing import check_signature + + conf = { + 'SECRET_KEY': 'the secret key', + 'API_KEY': 'the API key', + } + client = MirisManagerClient(conf) + + assert check_signature(client, signature) == expected