From 956ed9d67d5257226d0b6cc86d859be172e141cd Mon Sep 17 00:00:00 2001 From: Matthew John Date: Sat, 7 May 2022 16:13:16 +0100 Subject: [PATCH 1/5] fix(module-extractor): Issue #108 Catch error raised when performing git clone. Check for known git-style errors and raise back to user. Re-raise unknown exception if error cannot be detected. --- terrareg/errors.py | 6 ++++++ terrareg/module_extractor.py | 26 +++++++++++++++++++------- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/terrareg/errors.py b/terrareg/errors.py index 9fad7741..ee86aba0 100644 --- a/terrareg/errors.py +++ b/terrareg/errors.py @@ -148,3 +148,9 @@ class UnableToProcessTerraformError(TerraregError): """An error occurred whilst attempting to process terraform.""" pass + + +class GitCloneError(TerraregError): + """An error occurred during git clone.""" + + pass diff --git a/terrareg/module_extractor.py b/terrareg/module_extractor.py index fd114a31..d9b82b58 100644 --- a/terrareg/module_extractor.py +++ b/terrareg/module_extractor.py @@ -21,7 +21,8 @@ UnableToProcessTerraformError, UnknownFiletypeError, InvalidTerraregMetadataFileError, - MetadataDoesNotContainRequiredAttributeError + MetadataDoesNotContainRequiredAttributeError, + GitCloneError ) from terrareg.utils import PathDoesNotExistError, safe_join_paths from terrareg.config import Config @@ -299,12 +300,23 @@ def _clone_repository(self): if git_url.startswith('ssh://'): git_url = re.sub(r'^ssh://', '', git_url) - subprocess.check_call([ - 'git', 'clone', '--single-branch', - '--branch', self._module_version.source_git_tag, - git_url, - self.extract_directory - ], env=env) + try: + subprocess.check_output([ + 'git', 'clone', '--single-branch', + '--branch', self._module_version.source_git_tag, + git_url, + self.extract_directory + ], + stderr=subprocess.STDOUT, + # stdout=subprocess.PIPE, + env=env) + except subprocess.CalledProcessError as exc: + error = 'Unknown error occurred during git clone' + for line in exc.output.decode('ascii').split('\n'): + print(line) + if line.startswith('fatal:'): + error = 'Error occurred during git clone: {}'.format(line) + raise GitCloneError(error) def process_upload(self): """Extract archive and perform data extraction from module source.""" From 83eb26b536c69daea21d28324156f42845efa81b Mon Sep 17 00:00:00 2001 From: Matthew John Date: Sat, 7 May 2022 16:28:33 +0100 Subject: [PATCH 2/5] Issue #108 Improve formatting of git clone subprocess call --- terrareg/module_extractor.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/terrareg/module_extractor.py b/terrareg/module_extractor.py index d9b82b58..e894530c 100644 --- a/terrareg/module_extractor.py +++ b/terrareg/module_extractor.py @@ -302,14 +302,14 @@ def _clone_repository(self): try: subprocess.check_output([ - 'git', 'clone', '--single-branch', - '--branch', self._module_version.source_git_tag, - git_url, - self.extract_directory - ], - stderr=subprocess.STDOUT, - # stdout=subprocess.PIPE, - env=env) + 'git', 'clone', '--single-branch', + '--branch', self._module_version.source_git_tag, + git_url, + self.extract_directory + ], + stderr=subprocess.STDOUT, + env=env + ) except subprocess.CalledProcessError as exc: error = 'Unknown error occurred during git clone' for line in exc.output.decode('ascii').split('\n'): From e6721fb940ff1d713d1d2584cfb600f1097aab17 Mon Sep 17 00:00:00 2001 From: Matthew John Date: Sat, 7 May 2022 16:29:37 +0100 Subject: [PATCH 3/5] test: Issue #108 Fix tests for module_extractor to check for new subprocess call --- test/unit/terrareg/test_module_extractor.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/unit/terrareg/test_module_extractor.py b/test/unit/terrareg/test_module_extractor.py index 5c910a3b..3cf88064 100644 --- a/test/unit/terrareg/test_module_extractor.py +++ b/test/unit/terrareg/test_module_extractor.py @@ -1,4 +1,5 @@ +import subprocess from unittest.main import MODULE_EXAMPLES import unittest.mock @@ -32,7 +33,7 @@ def test__clone_repository(self, module_provider_name, expected_git_url, expecte check_call_mock = unittest.mock.MagicMock() module_extractor = GitModuleExtractor(module_version=module_version) - with unittest.mock.patch('terrareg.module_extractor.subprocess.check_call', check_call_mock): + with unittest.mock.patch('terrareg.module_extractor.subprocess.check_output', check_call_mock): with module_extractor as me: me._clone_repository() @@ -42,5 +43,6 @@ def test__clone_repository(self, module_provider_name, expected_git_url, expecte '--branch', expected_git_tag, expected_git_url, module_extractor.extract_directory], + stderr=subprocess.STDOUT, env=unittest.mock.ANY) assert check_call_mock.call_args.kwargs['env']['GIT_SSH_COMMAND'] == 'ssh -o StrictHostKeyChecking=accept-new' From 153da4748b2b3c3e9b84598cf7c6ed5aa66e7a12 Mon Sep 17 00:00:00 2001 From: Matthew John Date: Sat, 7 May 2022 16:30:48 +0100 Subject: [PATCH 4/5] test: Issue #108 Add tests to check exception raising when git throws an error. --- test/unit/terrareg/test_module_extractor.py | 45 +++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/test/unit/terrareg/test_module_extractor.py b/test/unit/terrareg/test_module_extractor.py index 3cf88064..0cbb759a 100644 --- a/test/unit/terrareg/test_module_extractor.py +++ b/test/unit/terrareg/test_module_extractor.py @@ -4,6 +4,7 @@ import unittest.mock import pytest +from terrareg.errors import GitCloneError from test.unit.terrareg import ( MockNamespace, MockModule, MockModuleProvider, @@ -46,3 +47,47 @@ def test__clone_repository(self, module_provider_name, expected_git_url, expecte stderr=subprocess.STDOUT, env=unittest.mock.ANY) assert check_call_mock.call_args.kwargs['env']['GIT_SSH_COMMAND'] == 'ssh -o StrictHostKeyChecking=accept-new' + + @setup_test_data() + def test_known_git_error(self): + """Test error thrown by git with expected format of error.""" + namespace = MockNamespace(name='moduleextraction') + module = MockModule(namespace=namespace, name='gitextraction') + module_provider = MockModuleProvider(module=module, name='staticrepourl') + module_version = MockModuleVersion(module_provider=module_provider, version='4.3.2') + + module_extractor = GitModuleExtractor(module_version=module_version) + + check_call_mock = unittest.mock.MagicMock() + test_error = subprocess.CalledProcessError(returncode=1, cmd=[]) + test_error.output = 'Preceeding line\nfatal: unittest error here\nend of output'.encode('ascii') + check_call_mock.side_effect = test_error + + with unittest.mock.patch('terrareg.module_extractor.subprocess.check_output', check_call_mock): + with module_extractor as me: + with pytest.raises(GitCloneError) as error: + me._clone_repository() + + assert str(error.value) == 'Error occurred during git clone: fatal: unittest error here' + + @setup_test_data() + def test_unknown_git_error(self): + """Test error thrown by git with expected format of error.""" + namespace = MockNamespace(name='moduleextraction') + module = MockModule(namespace=namespace, name='gitextraction') + module_provider = MockModuleProvider(module=module, name='staticrepourl') + module_version = MockModuleVersion(module_provider=module_provider, version='4.3.2') + + module_extractor = GitModuleExtractor(module_version=module_version) + + check_call_mock = unittest.mock.MagicMock() + test_error = subprocess.CalledProcessError(returncode=1, cmd=[]) + test_error.output = 'Preceeding line\nnot a recognised output\nend of output'.encode('ascii') + check_call_mock.side_effect = test_error + + with unittest.mock.patch('terrareg.module_extractor.subprocess.check_output', check_call_mock): + with module_extractor as me: + with pytest.raises(GitCloneError) as error: + me._clone_repository() + + assert str(error.value) == 'Unknown error occurred during git clone' From 2722c2d7f609b4bbea3c8e48188c275116ef88b8 Mon Sep 17 00:00:00 2001 From: Matthew John Date: Sat, 7 May 2022 16:40:47 +0100 Subject: [PATCH 5/5] chore(ci): Issue #108 Install requirements.txt in build environment. Tests are failing due to alembic not being available. --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 34d8bf44..87e270ff 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -27,6 +27,7 @@ cache: - pip install --proxy=$http_proxy virtualenv - virtualenv venv - source venv/bin/activate + - pip install --proxy=$http_proxy -r requirements.txt unit-tests: stage: test