EES-5017 Create indicators Parquet meta table

dfe-analytical-services · Apr 3, 2024 · 5ad3eca · 5ad3eca
1 parent ca35159
commit 5ad3eca
Show file tree

Hide file tree

Showing 8 changed files with 97 additions and 2 deletions.
diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DataSetVersion.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DataSetVersion.cs
@@ -69,6 +69,8 @@ public DataSetVersionType VersionType
 
     public string FiltersParquetPath => ParquetPaths.FiltersPath(this);
 
+    public string IndicatorsParquetPath  => ParquetPaths.IndicatorsPath(this);
+
     public string LocationsParquetPath => ParquetPaths.LocationsPath(this);
 
     public string TimePeriodsParquetPath => ParquetPaths.TimePeriodsPath(this);

diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/Parquet/ParquetIndicator.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/Parquet/ParquetIndicator.cs
@@ -0,0 +1,12 @@
+namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Parquet;
+
+public class ParquetIndicator
+{
+    public required string Id { get; set; }
+
+    public required string Label { get; set; }
+
+    public string Unit { get; set; } = string.Empty;
+
+    public byte DecimalPlaces { get; set; }
+}
diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/Parquet/ParquetPaths.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/Parquet/ParquetPaths.cs
@@ -28,4 +28,9 @@ public static string TimePeriodsPath(DataSetVersion dataSetVersion) => Path.Comb
         DirectoryPath(dataSetVersion),
         $"{TimePeriodsTable.TableName}.parquet"
     );
+
+    public static string IndicatorsPath(DataSetVersion dataSetVersion) => Path.Combine(
+        DirectoryPath(dataSetVersion),
+        $"{IndicatorsTable.TableName}.parquet"
+    );
 }
diff --git a/....Education.ExploreEducationStatistics.Public.Data.Model/Parquet/Tables/IndicatorsTable.cs b/....Education.ExploreEducationStatistics.Public.Data.Model/Parquet/Tables/IndicatorsTable.cs
@@ -0,0 +1,31 @@
+namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Parquet.Tables;
+
+public static class IndicatorsTable
+{
+    public const string TableName = "indicators";
+
+    public static class Cols
+    {
+        public const string Id = "id";
+        public const string Label = "label";
+        public const string Unit = "unit";
+        public const string DecimalPlaces = "decimal_places";
+    }
+
+    public static string Alias(IndicatorMeta indicator) => $"\"{indicator.PublicId}\"";
+
+    private static readonly TableRef DefaultRef = new(TableName);
+
+    public static TableRef Ref(IndicatorMeta? indicator = null)
+        => indicator is not null ? new(Alias(indicator)) : DefaultRef;
+
+    public class TableRef(string table)
+    {
+        public readonly string Id = $"{table}.{Cols.Id}";
+        public readonly string Label = $"{table}.{Cols.Label}";
+        public readonly string Unit = $"{table}.{Cols.Unit}";
+        public readonly string DecimalPlaces = $"{table}.{Cols.DecimalPlaces}";
+
+        public string Col(string column) => $"{table}.\"{column}\"";
+    }
+}
diff --git a/...ovUk.Education.ExploreEducationStatistics.Public.Data.Scripts/Commands/SeedDataCommand.cs b/...ovUk.Education.ExploreEducationStatistics.Public.Data.Scripts/Commands/SeedDataCommand.cs
@@ -855,12 +855,40 @@ await _duckDb.ExecuteAsync(
 
         private async Task CreateParquetMetaTables(DataSetVersion version)
         {
-            await CreateParquetLocationMetaTables(version);
+            await CreateParquetIndicatorTable(version);
+            await CreateParquetLocationMetaTable(version);
             await CreateParquetFilterMetaTable(version);
             await CreateParquetTimePeriodMetaTable(version);
         }
 
-        private async Task CreateParquetLocationMetaTables(DataSetVersion version)
+        private async Task CreateParquetIndicatorTable(DataSetVersion version)
+        {
+            await _duckDb.ExecuteAsync(
+                $"""
+                 CREATE TABLE {IndicatorsTable.TableName}(
+                     {IndicatorsTable.Cols.Id} VARCHAR PRIMARY KEY,
+                     {IndicatorsTable.Cols.Label} VARCHAR,
+                     {IndicatorsTable.Cols.Unit} VARCHAR,
+                     {IndicatorsTable.Cols.DecimalPlaces} TINYINT,
+                 )
+                 """
+            );
+
+            using var appender = _duckDb.CreateAppender(table: IndicatorsTable.TableName);
+
+            foreach (var meta in version.IndicatorMetas)
+            {
+                var insertRow = appender.CreateRow();
+
+                insertRow.AppendValue(meta.PublicId);
+                insertRow.AppendValue(meta.Label);
+                insertRow.AppendValue(meta.Unit?.GetEnumLabel() ?? string.Empty);
+                insertRow.AppendValue(meta.DecimalPlaces);
+                insertRow.EndRow();
+            }
+        }
+
+        private async Task CreateParquetLocationMetaTable(DataSetVersion version)
         {
             await _duckDb.ExecuteAsync(
                 $"""

diff --git a/...ucation.ExploreEducationStatistics.Public.Data.Services.Tests/ParquetPathResolverTests.cs b/...ucation.ExploreEducationStatistics.Public.Data.Services.Tests/ParquetPathResolverTests.cs
@@ -68,6 +68,10 @@ public void DevelopmentEnv_ValidBasePath_AllPathsCorrect()
                 Path.Combine(expectedBasePath, version.FiltersParquetPath),
                 resolver.FiltersPath(version)
             );
+            Assert.Equal(
+                Path.Combine(expectedBasePath, version.IndicatorsParquetPath),
+                resolver.IndicatorsPath(version)
+            );
             Assert.Equal(
                 Path.Combine(expectedBasePath, version.LocationsParquetPath),
                 resolver.LocationsPath(version)
@@ -110,6 +114,10 @@ public void IntegrationTestEnv_ValidBasePath_AllPathsCorrect()
                 Path.Combine(expectedBasePath, version.FiltersParquetPath),
                 resolver.FiltersPath(version)
             );
+            Assert.Equal(
+                Path.Combine(expectedBasePath, version.IndicatorsParquetPath),
+                resolver.IndicatorsPath(version)
+            );
             Assert.Equal(
                 Path.Combine(expectedBasePath, version.LocationsParquetPath),
                 resolver.LocationsPath(version)
@@ -151,6 +159,10 @@ public void ProductionEnv_ValidBasePath_AllPathsCorrect()
                 Path.Combine(expectedBasePath, version.FiltersParquetPath),
                 resolver.FiltersPath(version)
             );
+            Assert.Equal(
+                Path.Combine(expectedBasePath, version.IndicatorsParquetPath),
+                resolver.IndicatorsPath(version)
+            );
             Assert.Equal(
                 Path.Combine(expectedBasePath, version.LocationsParquetPath),
                 resolver.LocationsPath(version)

diff --git a/...cation.ExploreEducationStatistics.Public.Data.Services/Interfaces/IParquetPathResolver.cs b/...cation.ExploreEducationStatistics.Public.Data.Services/Interfaces/IParquetPathResolver.cs
@@ -10,6 +10,8 @@ public interface IParquetPathResolver
 
     string FiltersPath(DataSetVersion dataSetVersion);
 
+    string IndicatorsPath(DataSetVersion dataSetVersion);
+
     string LocationsPath(DataSetVersion dataSetVersion);
 
     string TimePeriodsPath(DataSetVersion dataSetVersion);

diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Services/ParquetPathResolver.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Services/ParquetPathResolver.cs
@@ -42,6 +42,9 @@ public string DataPath(DataSetVersion dataSetVersion)
     public string FiltersPath(DataSetVersion dataSetVersion)
         => Path.Combine(_basePath, dataSetVersion.FiltersParquetPath);
 
+    public string IndicatorsPath(DataSetVersion dataSetVersion)
+        => Path.Combine(_basePath, dataSetVersion.IndicatorsParquetPath);
+
     public string LocationsPath(DataSetVersion dataSetVersion)
         => Path.Combine(_basePath, dataSetVersion.LocationsParquetPath);