Merge branch '107-add-required-packages-to-connect-to-mysql-databae' …

…into 'main'

Resolve "Add required packages to connect to MySQL database"

Closes #107

See merge request pub/terrareg!78

README.md

@@ -419,9 +419,17 @@ There are common attributes that can be added to each of variable objects, which
 
 ## Changelog
 
+### v2.0.0
+
+WARNING: This version does not support migration from previous versions.
+
+ * Update all database columns to use MySQL-compatible types.
+ * Convert large data values to blobs
+
 ### v1.1.0
 
  * Add MySQL connector and document URL format to connect to MySQL
+ * Fix SQL schema to work with mysql
  * Provide ability to pass SSH private key through environment variable
 
 ### v1.0.3

terrareg/alembic/versions/b6610a495617_.py → terrareg/alembic/versions/aef5947a7e1d_.py

@@ -1,16 +1,16 @@
 """empty message
 
-Revision ID: b6610a495617
+Revision ID: aef5947a7e1d
 Revises: 
-Create Date: 2022-04-29 06:23:26.027687
+Create Date: 2022-05-06 13:09:10.827263
 
 """
 from alembic import op
 import sqlalchemy as sa
 
 
 # revision identifiers, used by Alembic.
-revision = 'b6610a495617'
+revision = 'aef5947a7e1d'
 down_revision = None
 branch_labels = None
 depends_on = None
@@ -20,22 +20,22 @@ def upgrade():
     # ### commands auto generated by Alembic - please adjust! ###
     op.create_table('git_provider',
     sa.Column('id', sa.Integer(), nullable=False),
-    sa.Column('name', sa.String(), nullable=True),
-    sa.Column('base_url_template', sa.String(), nullable=True),
-    sa.Column('clone_url_template', sa.String(), nullable=True),
-    sa.Column('browse_url_template', sa.String(), nullable=True),
+    sa.Column('name', sa.String(length=1024), nullable=True),
+    sa.Column('base_url_template', sa.String(length=1024), nullable=True),
+    sa.Column('clone_url_template', sa.String(length=1024), nullable=True),
+    sa.Column('browse_url_template', sa.String(length=1024), nullable=True),
     sa.PrimaryKeyConstraint('id'),
     sa.UniqueConstraint('name')
     )
     op.create_table('module_provider',
     sa.Column('id', sa.Integer(), nullable=False),
-    sa.Column('namespace', sa.String(), nullable=True),
-    sa.Column('module', sa.String(), nullable=True),
-    sa.Column('provider', sa.String(), nullable=True),
-    sa.Column('repo_base_url_template', sa.String(), nullable=True),
-    sa.Column('repo_clone_url_template', sa.String(), nullable=True),
-    sa.Column('repo_browse_url_template', sa.String(), nullable=True),
-    sa.Column('git_tag_format', sa.String(), nullable=True),
+    sa.Column('namespace', sa.String(length=1024), nullable=True),
+    sa.Column('module', sa.String(length=1024), nullable=True),
+    sa.Column('provider', sa.String(length=1024), nullable=True),
+    sa.Column('repo_base_url_template', sa.String(length=1024), nullable=True),
+    sa.Column('repo_clone_url_template', sa.String(length=1024), nullable=True),
+    sa.Column('repo_browse_url_template', sa.String(length=1024), nullable=True),
+    sa.Column('git_tag_format', sa.String(length=1024), nullable=True),
     sa.Column('verified', sa.Boolean(), nullable=True),
     sa.Column('git_provider_id', sa.Integer(), nullable=True),
     sa.ForeignKeyConstraint(['git_provider_id'], ['git_provider.id'], onupdate='CASCADE', ondelete='SET NULL'),
@@ -44,39 +44,39 @@ def upgrade():
     op.create_table('module_version',
     sa.Column('id', sa.Integer(), nullable=False),
     sa.Column('module_provider_id', sa.Integer(), nullable=False),
-    sa.Column('version', sa.String(), nullable=True),
-    sa.Column('owner', sa.String(), nullable=True),
-    sa.Column('description', sa.String(), nullable=True),
-    sa.Column('repo_base_url_template', sa.String(), nullable=True),
-    sa.Column('repo_clone_url_template', sa.String(), nullable=True),
-    sa.Column('repo_browse_url_template', sa.String(), nullable=True),
+    sa.Column('version', sa.String(length=1024), nullable=True),
+    sa.Column('owner', sa.String(length=1024), nullable=True),
+    sa.Column('description', sa.String(length=1024), nullable=True),
+    sa.Column('repo_base_url_template', sa.String(length=1024), nullable=True),
+    sa.Column('repo_clone_url_template', sa.String(length=1024), nullable=True),
+    sa.Column('repo_browse_url_template', sa.String(length=1024), nullable=True),
     sa.Column('published_at', sa.DateTime(), nullable=True),
-    sa.Column('readme_content', sa.String(), nullable=True),
-    sa.Column('module_details', sa.String(), nullable=True),
-    sa.Column('variable_template', sa.String(), nullable=True),
-    sa.Column('artifact_location', sa.String(), nullable=True),
+    sa.Column('readme_content', sa.BLOB(), nullable=True),
+    sa.Column('module_details', sa.BLOB(), nullable=True),
+    sa.Column('variable_template', sa.BLOB(), nullable=True),
     sa.Column('published', sa.Boolean(), nullable=True),
     sa.ForeignKeyConstraint(['module_provider_id'], ['module_provider.id'], onupdate='CASCADE', ondelete='CASCADE'),
     sa.PrimaryKeyConstraint('id')
     )
     op.create_table('analytics',
     sa.Column('id', sa.Integer(), nullable=False),
     sa.Column('parent_module_version', sa.Integer(), nullable=False),
-    sa.Column('timestamp', sa.String(), nullable=True),
-    sa.Column('terraform_version', sa.String(), nullable=True),
-    sa.Column('analytics_token', sa.String(), nullable=True),
-    sa.Column('auth_token', sa.String(), nullable=True),
-    sa.Column('environment', sa.String(), nullable=True),
+    sa.Column('timestamp', sa.DateTime(), nullable=True),
+    sa.Column('terraform_version', sa.String(length=1024), nullable=True),
+    sa.Column('analytics_token', sa.String(length=1024), nullable=True),
+    sa.Column('auth_token', sa.String(length=1024), nullable=True),
+    sa.Column('environment', sa.String(length=1024), nullable=True),
     sa.ForeignKeyConstraint(['parent_module_version'], ['module_version.id'], onupdate='CASCADE', ondelete='CASCADE'),
     sa.PrimaryKeyConstraint('id')
     )
     op.create_table('submodule',
     sa.Column('id', sa.Integer(), nullable=False),
     sa.Column('parent_module_version', sa.Integer(), nullable=False),
-    sa.Column('path', sa.String(), nullable=True),
-    sa.Column('name', sa.String(), nullable=True),
-    sa.Column('readme_content', sa.String(), nullable=True),
-    sa.Column('module_details', sa.String(), nullable=True),
+    sa.Column('type', sa.String(length=1024), nullable=True),
+    sa.Column('path', sa.String(length=1024), nullable=True),
+    sa.Column('name', sa.String(length=1024), nullable=True),
+    sa.Column('readme_content', sa.BLOB(), nullable=True),
+    sa.Column('module_details', sa.BLOB(), nullable=True),
     sa.ForeignKeyConstraint(['parent_module_version'], ['module_version.id'], onupdate='CASCADE', ondelete='CASCADE'),
     sa.PrimaryKeyConstraint('id')
     )

terrareg/database.py

@@ -13,6 +13,21 @@ class Database():
     _ENGINE = None
     _INSTANCE = None
 
+    blob_encoding_format = 'utf-8'
+
+    @staticmethod
+    def encode_blob(value):
+        """Encode string as a blog value"""
+        # Convert any untruthful values to empty string
+        if not value:
+            value = ''
+        return value.encode(Database.blob_encoding_format)
+
+    @staticmethod
+    def decode_blob(value):
+        """Decode blob as a string."""
+        return value.decode(Database.blob_encoding_format)
+
     def __init__(self):
         """Setup member variables."""
         self._git_provider = None
@@ -91,25 +106,27 @@ def initialise(self):
         meta = self.get_meta()
         engine = self.get_engine()
 
+        GENERAL_COLUMN_SIZE = 1024
+
         self._git_provider = sqlalchemy.Table(
             'git_provider', meta,
             sqlalchemy.Column('id', sqlalchemy.Integer, primary_key = True),
-            sqlalchemy.Column('name', sqlalchemy.String, unique=True),
-            sqlalchemy.Column('base_url_template', sqlalchemy.String),
-            sqlalchemy.Column('clone_url_template', sqlalchemy.String),
-            sqlalchemy.Column('browse_url_template', sqlalchemy.String)
+            sqlalchemy.Column('name', sqlalchemy.String(GENERAL_COLUMN_SIZE), unique=True),
+            sqlalchemy.Column('base_url_template', sqlalchemy.String(GENERAL_COLUMN_SIZE)),
+            sqlalchemy.Column('clone_url_template', sqlalchemy.String(GENERAL_COLUMN_SIZE)),
+            sqlalchemy.Column('browse_url_template', sqlalchemy.String(GENERAL_COLUMN_SIZE))
         )
 
         self._module_provider = sqlalchemy.Table(
             'module_provider', meta,
             sqlalchemy.Column('id', sqlalchemy.Integer, primary_key = True),
-            sqlalchemy.Column('namespace', sqlalchemy.String),
-            sqlalchemy.Column('module', sqlalchemy.String),
-            sqlalchemy.Column('provider', sqlalchemy.String),
-            sqlalchemy.Column('repo_base_url_template', sqlalchemy.String),
-            sqlalchemy.Column('repo_clone_url_template', sqlalchemy.String),
-            sqlalchemy.Column('repo_browse_url_template', sqlalchemy.String),
-            sqlalchemy.Column('git_tag_format', sqlalchemy.String),
+            sqlalchemy.Column('namespace', sqlalchemy.String(GENERAL_COLUMN_SIZE)),
+            sqlalchemy.Column('module', sqlalchemy.String(GENERAL_COLUMN_SIZE)),
+            sqlalchemy.Column('provider', sqlalchemy.String(GENERAL_COLUMN_SIZE)),
+            sqlalchemy.Column('repo_base_url_template', sqlalchemy.String(GENERAL_COLUMN_SIZE)),
+            sqlalchemy.Column('repo_clone_url_template', sqlalchemy.String(GENERAL_COLUMN_SIZE)),
+            sqlalchemy.Column('repo_browse_url_template', sqlalchemy.String(GENERAL_COLUMN_SIZE)),
+            sqlalchemy.Column('git_tag_format', sqlalchemy.String(GENERAL_COLUMN_SIZE)),
             sqlalchemy.Column('verified', sqlalchemy.Boolean),
             sqlalchemy.Column(
                 'git_provider_id',
@@ -132,16 +149,16 @@ def initialise(self):
                     ondelete='CASCADE'),
                 nullable=False
             ),
-            sqlalchemy.Column('version', sqlalchemy.String),
-            sqlalchemy.Column('owner', sqlalchemy.String),
-            sqlalchemy.Column('description', sqlalchemy.String),
-            sqlalchemy.Column('repo_base_url_template', sqlalchemy.String),
-            sqlalchemy.Column('repo_clone_url_template', sqlalchemy.String),
-            sqlalchemy.Column('repo_browse_url_template', sqlalchemy.String),
+            sqlalchemy.Column('version', sqlalchemy.String(GENERAL_COLUMN_SIZE)),
+            sqlalchemy.Column('owner', sqlalchemy.String(GENERAL_COLUMN_SIZE)),
+            sqlalchemy.Column('description', sqlalchemy.String(GENERAL_COLUMN_SIZE)),
+            sqlalchemy.Column('repo_base_url_template', sqlalchemy.String(GENERAL_COLUMN_SIZE)),
+            sqlalchemy.Column('repo_clone_url_template', sqlalchemy.String(GENERAL_COLUMN_SIZE)),
+            sqlalchemy.Column('repo_browse_url_template', sqlalchemy.String(GENERAL_COLUMN_SIZE)),
             sqlalchemy.Column('published_at', sqlalchemy.DateTime),
-            sqlalchemy.Column('readme_content', sqlalchemy.String),
-            sqlalchemy.Column('module_details', sqlalchemy.String),
-            sqlalchemy.Column('variable_template', sqlalchemy.String),
+            sqlalchemy.Column('readme_content', sqlalchemy.BLOB),
+            sqlalchemy.Column('module_details', sqlalchemy.BLOB),
+            sqlalchemy.Column('variable_template', sqlalchemy.BLOB),
             sqlalchemy.Column('published', sqlalchemy.Boolean)
         )
 
@@ -156,11 +173,11 @@ def initialise(self):
                     ondelete='CASCADE'),
                 nullable=False
             ),
-            sqlalchemy.Column('type', sqlalchemy.String),
-            sqlalchemy.Column('path', sqlalchemy.String),
-            sqlalchemy.Column('name', sqlalchemy.String),
-            sqlalchemy.Column('readme_content', sqlalchemy.String),
-            sqlalchemy.Column('module_details', sqlalchemy.String)
+            sqlalchemy.Column('type', sqlalchemy.String(GENERAL_COLUMN_SIZE)),
+            sqlalchemy.Column('path', sqlalchemy.String(GENERAL_COLUMN_SIZE)),
+            sqlalchemy.Column('name', sqlalchemy.String(GENERAL_COLUMN_SIZE)),
+            sqlalchemy.Column('readme_content', sqlalchemy.BLOB),
+            sqlalchemy.Column('module_details', sqlalchemy.BLOB)
         )
 
         self._analytics = sqlalchemy.Table(
@@ -174,11 +191,11 @@ def initialise(self):
                     ondelete='CASCADE'),
                 nullable=False
             ),
-            sqlalchemy.Column('timestamp', sqlalchemy.String),
-            sqlalchemy.Column('terraform_version', sqlalchemy.String),
-            sqlalchemy.Column('analytics_token', sqlalchemy.String),
-            sqlalchemy.Column('auth_token', sqlalchemy.String),
-            sqlalchemy.Column('environment', sqlalchemy.String)
+            sqlalchemy.Column('timestamp', sqlalchemy.DateTime),
+            sqlalchemy.Column('terraform_version', sqlalchemy.String(GENERAL_COLUMN_SIZE)),
+            sqlalchemy.Column('analytics_token', sqlalchemy.String(GENERAL_COLUMN_SIZE)),
+            sqlalchemy.Column('auth_token', sqlalchemy.String(GENERAL_COLUMN_SIZE)),
+            sqlalchemy.Column('environment', sqlalchemy.String(GENERAL_COLUMN_SIZE))
         )
 
     def select_module_version_joined_module_provider(self, *args, **kwargs):

terrareg/models.py

@@ -855,7 +855,7 @@ def _get_db_row(self):
     def get_module_specs(self):
         """Return module specs"""
         if self._module_specs is None:
-            self._module_specs = json.loads(self._get_db_row()['module_details'])
+            self._module_specs = json.loads(Database.decode_blob(self._get_db_row()['module_details']))
         return self._module_specs
 
     def get_readme_html(self):
@@ -871,7 +871,7 @@ def get_readme_html(self):
 
     def get_readme_content(self):
         """Get readme contents"""
-        return self._get_db_row()['readme_content']
+        return Database.decode_blob(self._get_db_row()['readme_content'])
 
     def get_terraform_inputs(self):
         """Obtain module inputs"""
@@ -1053,7 +1053,7 @@ def registry_id(self):
     @property
     def variable_template(self):
         """Return variable template for module version."""
-        return json.loads(self._get_db_row()['variable_template'])
+        return json.loads(Database.decode_blob(self._get_db_row()['variable_template']))
 
     def __init__(self, module_provider: ModuleProvider, version: str):
         """Setup member variables."""
@@ -1261,6 +1261,11 @@ def get_db_where(self, db, statement):
 
     def update_attributes(self, **kwargs):
         """Update attributes of module version in database row."""
+        # Check for any blob and encode the values
+        for kwarg in kwargs:
+            if kwarg in ['readme_content', 'module_details', 'variable_template']:
+                kwargs[kwarg] = Database.encode_blob(kwargs[kwarg])
+
         db = Database.get()
         update = self.get_db_where(
             db=db, statement=db.module_version.update()

terrareg/module_extractor.py

@@ -159,8 +159,8 @@ def _process_submodule(self, type_: str, submodule: str):
             parent_module_version=self._module_version.pk,
             type=type_,
             path=submodule,
-            readme_content=readme_content,
-            module_details=json.dumps(tf_docs)
+            readme_content=Database.encode_blob(readme_content),
+            module_details=Database.encode_blob(json.dumps(tf_docs))
         )
         conn.execute(insert_statement)
 

test/unit/terrareg/__init__.py

@@ -127,12 +127,12 @@ def _get_db_row(self):
                 datetime.datetime(year=2020, month=1, day=1,
                                   hour=23, minute=18, second=12)
             ),
-            'readme_content': self._unittest_data.get('readme_content', 'Mock module README file'),
-            'module_details': self._unittest_data.get(
+            'readme_content': Database.encode_blob(self._unittest_data.get('readme_content', 'Mock module README file')),
+            'module_details': Database.encode_blob(self._unittest_data.get(
                 'module_details',
                 '{"inputs": [], "outputs": [], "providers": [], "resources": []}'
-            ),
-            'variable_template': self._unittest_data.get('variable_template', '{}')
+            )),
+            'variable_template': Database.encode_blob(self._unittest_data.get('variable_template', '{}'))
         }