Merge branch '108-provide-better-errors-when-cloning-url' into 'main'

Resolve "Provide better errors when cloning  URL"

Closes #108

See merge request pub/terrareg!83

.gitlab-ci.yml

@@ -27,6 +27,7 @@ cache:
     - pip install --proxy=$http_proxy virtualenv
     - virtualenv venv
     - source venv/bin/activate
+    - pip install --proxy=$http_proxy -r requirements.txt
 
 unit-tests:
   stage: test

terrareg/errors.py

@@ -148,3 +148,9 @@ class UnableToProcessTerraformError(TerraregError):
     """An error occurred whilst attempting to process terraform."""
 
     pass
+
+
+class GitCloneError(TerraregError):
+    """An error occurred during git clone."""
+
+    pass

terrareg/module_extractor.py

@@ -21,7 +21,8 @@
     UnableToProcessTerraformError,
     UnknownFiletypeError,
     InvalidTerraregMetadataFileError,
-    MetadataDoesNotContainRequiredAttributeError
+    MetadataDoesNotContainRequiredAttributeError,
+    GitCloneError
 )
 from terrareg.utils import PathDoesNotExistError, safe_join_paths
 from terrareg.config import Config
@@ -299,12 +300,23 @@ def _clone_repository(self):
         if git_url.startswith('ssh://'):
             git_url = re.sub(r'^ssh://', '', git_url)
 
-        subprocess.check_call([
-            'git', 'clone', '--single-branch',
-            '--branch', self._module_version.source_git_tag,
-            git_url,
-            self.extract_directory
-        ], env=env)
+        try:
+            subprocess.check_output([
+                    'git', 'clone', '--single-branch',
+                    '--branch', self._module_version.source_git_tag,
+                    git_url,
+                    self.extract_directory
+                ],
+                stderr=subprocess.STDOUT,
+                env=env
+            )
+        except subprocess.CalledProcessError as exc:
+            error = 'Unknown error occurred during git clone'
+            for line in exc.output.decode('ascii').split('\n'):
+                print(line)
+                if line.startswith('fatal:'):
+                    error = 'Error occurred during git clone: {}'.format(line)
+            raise GitCloneError(error)
 
     def process_upload(self):
         """Extract archive and perform data extraction from module source."""

test/unit/terrareg/test_module_extractor.py

@@ -1,8 +1,10 @@
 
+import subprocess
 from unittest.main import MODULE_EXAMPLES
 import unittest.mock
 
 import pytest
+from terrareg.errors import GitCloneError
 
 from test.unit.terrareg import (
     MockNamespace, MockModule, MockModuleProvider,
@@ -32,7 +34,7 @@ def test__clone_repository(self, module_provider_name, expected_git_url, expecte
 
         check_call_mock = unittest.mock.MagicMock()
         module_extractor = GitModuleExtractor(module_version=module_version)
-        with unittest.mock.patch('terrareg.module_extractor.subprocess.check_call', check_call_mock):
+        with unittest.mock.patch('terrareg.module_extractor.subprocess.check_output', check_call_mock):
             with module_extractor as me:
                 me._clone_repository()
 
@@ -42,5 +44,50 @@ def test__clone_repository(self, module_provider_name, expected_git_url, expecte
             '--branch', expected_git_tag,
             expected_git_url,
             module_extractor.extract_directory],
+            stderr=subprocess.STDOUT,
             env=unittest.mock.ANY)
         assert check_call_mock.call_args.kwargs['env']['GIT_SSH_COMMAND'] == 'ssh -o StrictHostKeyChecking=accept-new'
+
+    @setup_test_data()
+    def test_known_git_error(self):
+        """Test error thrown by git with expected format of error."""
+        namespace = MockNamespace(name='moduleextraction')
+        module = MockModule(namespace=namespace, name='gitextraction')
+        module_provider = MockModuleProvider(module=module, name='staticrepourl')
+        module_version = MockModuleVersion(module_provider=module_provider, version='4.3.2')
+
+        module_extractor = GitModuleExtractor(module_version=module_version)
+
+        check_call_mock = unittest.mock.MagicMock()
+        test_error = subprocess.CalledProcessError(returncode=1, cmd=[])
+        test_error.output = 'Preceeding line\nfatal: unittest error here\nend of output'.encode('ascii')
+        check_call_mock.side_effect = test_error
+
+        with unittest.mock.patch('terrareg.module_extractor.subprocess.check_output', check_call_mock):
+            with module_extractor as me:
+                with pytest.raises(GitCloneError) as error:
+                    me._clone_repository()
+
+                assert str(error.value) == 'Error occurred during git clone: fatal: unittest error here'
+
+    @setup_test_data()
+    def test_unknown_git_error(self):
+        """Test error thrown by git with expected format of error."""
+        namespace = MockNamespace(name='moduleextraction')
+        module = MockModule(namespace=namespace, name='gitextraction')
+        module_provider = MockModuleProvider(module=module, name='staticrepourl')
+        module_version = MockModuleVersion(module_provider=module_provider, version='4.3.2')
+
+        module_extractor = GitModuleExtractor(module_version=module_version)
+
+        check_call_mock = unittest.mock.MagicMock()
+        test_error = subprocess.CalledProcessError(returncode=1, cmd=[])
+        test_error.output = 'Preceeding line\nnot a recognised output\nend of output'.encode('ascii')
+        check_call_mock.side_effect = test_error
+
+        with unittest.mock.patch('terrareg.module_extractor.subprocess.check_output', check_call_mock):
+            with module_extractor as me:
+                with pytest.raises(GitCloneError) as error:
+                    me._clone_repository()
+
+                assert str(error.value) == 'Unknown error occurred during git clone'