Expose NetCDF quantize and zstandard functions

.github/workflows/nightly-build.yml

@@ -74,9 +74,9 @@ jobs:
           export DST_META_YAML=`pwd`/$PROJECT_DIR/cmor-feedstock/recipe/meta.yaml
           mv $DST_META_YAML $SRC_META_YAML
 
-          export GIT_REV=$(git rev-parse --short HEAD)
+          export GIT_REV=$(git rev-parse --short "$GITHUB_SHA")
           echo "GIT_REV=$GIT_REV"
-          export GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
+          export GIT_BRANCH=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}
           echo "GIT_BRANCH=$GIT_BRANCH"
 
           python rebuild_meta_yaml.py \

Lib/__init__.py

@@ -11,8 +11,9 @@
     close, grid, set_grid_mapping, time_varying_grid_coordinate, dataset_json,
     set_cur_dataset_attribute, get_cur_dataset_attribute,
     has_cur_dataset_attribute, set_variable_attribute, get_variable_attribute,
-    has_variable_attribute, get_final_filename, set_deflate, set_furtherinfourl, 
-    set_climatology, get_climatology, set_terminate_signal, get_terminate_signal)
+    has_variable_attribute, get_final_filename, set_deflate, set_zstandard,
+    set_quantize, set_furtherinfourl, set_climatology, get_climatology,
+    set_terminate_signal, get_terminate_signal)
 
 try:
     from check_CMOR_compliant import checkCMOR

Lib/pywrapper.py

@@ -1032,6 +1032,41 @@ def set_deflate(var_id, shuffle, deflate, deflate_level):
     return _cmor.set_deflate(var_id, shuffle, deflate, deflate_level)
 
 
+def set_zstandard(var_id, zstandar_level):
+    """Sets Zstandard compression on a cmor variable
+    Usage:
+      cmor.set_zstandard(var_id, szstandar_level)
+    Where:
+      var_id: is cmor variable id
+      zstandar_level: Compression level. Must be set from -131072 to 22
+
+    """
+
+    return _cmor.set_zstandard(var_id, zstandar_level)
+
+
+def set_quantize(var_id, quantize_mode, quantize_nsd):
+    """Sets quantization on a cmor variable
+    Usage:
+      cmor.set_quantize(var_id, quantize_mode, quantize_nsd)
+    Where:
+      var_id: is cmor variable id
+      quantize_mode: Quantization mode.  Can be set to the following.
+                     0: No quantization mode
+                     1: BitGroom
+                     2: Granular BitRound
+                     3: BitRound
+      quantize_nsd: Number of significant digits.  If quantize_mode is set to
+                    1 or 2, then the value can be set from 1 to 7 for floats
+                    and 1 to 23 for doubles. If quantize_mode is set to 3, then
+                    the value can be set from 1 to 15 for floats and 1 to 52
+                    for doubles.  The value is ignore if quantize_mode is 0.
+
+    """
+
+    return _cmor.set_quantize(var_id, quantize_mode, quantize_nsd)
+
+
 def has_variable_attribute(var_id, name):
     """determines if the a cmor variable has an attribute
     Usage:

Makefile.in

@@ -225,6 +225,7 @@ test_python: python
 	env TEST_NAME=Test/test_cmor_python_not_enough_times_written.py make test_a_python
 	env TEST_NAME=Test/test_python_forecast_coordinates.py make test_a_python
 	env TEST_NAME=Test/test_cmor_CMIP6Plus.py make test_a_python
+	env TEST_NAME=Test/test_cmor_zstandard_and_quantize.py make test_a_python
 test_cmip6_cv: python
 	env TEST_NAME=Test/test_python_CMIP6_CV_sub_experimentnotset.py make test_a_python
 	env TEST_NAME=Test/test_python_CMIP6_CV_sub_experimentbad.py make test_a_python

Src/_cmormodule.c

@@ -151,6 +151,52 @@ static PyObject *PyCMOR_set_deflate(PyObject * self, PyObject * args)
     return (Py_BuildValue("i", ierr));
 }
 
+/************************************************************************/
+/*                   PyCMOR_set_zstandard()                    */
+/************************************************************************/
+static PyObject *PyCMOR_set_zstandard(PyObject * self, PyObject * args)
+{
+    signal(signal_to_catch, signal_handler);
+    int ierr, var_id, zstandard_level;
+
+    if (!PyArg_ParseTuple
+        (args, "ii", &var_id, &zstandard_level))
+        return NULL;
+
+    ierr = cmor_set_zstandard(var_id, zstandard_level);
+
+    if (ierr != 0 || raise_exception) {
+        raise_exception = 0;
+        PyErr_Format(CMORError, exception_message, "set_zstandard");
+        return NULL;
+    }
+
+    return (Py_BuildValue("i", ierr));
+}
+
+/************************************************************************/
+/*                   PyCMOR_set_quantize()                    */
+/************************************************************************/
+static PyObject *PyCMOR_set_quantize(PyObject * self, PyObject * args)
+{
+    signal(signal_to_catch, signal_handler);
+    int ierr, var_id, quantize_mode, quantize_nsd;
+
+    if (!PyArg_ParseTuple
+        (args, "iii", &var_id, &quantize_mode, &quantize_nsd))
+        return NULL;
+
+    ierr = cmor_set_quantize(var_id, quantize_mode, quantize_nsd);
+
+    if (ierr != 0 || raise_exception) {
+        raise_exception = 0;
+        PyErr_Format(CMORError, exception_message, "set_quantize");
+        return NULL;
+    }
+
+    return (Py_BuildValue("i", ierr));
+}
+
 /************************************************************************/
 /*                   PyCMOR_set_variable_attribute()                    */
 /************************************************************************/
@@ -1146,6 +1192,8 @@ static PyMethodDef MyExtractMethods[] = {
     {"set_furtherinfourl", PyCMOR_set_furtherinfourl, METH_VARARGS},
     {"get_final_filename", PyCMOR_getFinalFilename, METH_VARARGS},
     {"set_deflate", PyCMOR_set_deflate, METH_VARARGS},
+    {"set_zstandard", PyCMOR_set_zstandard, METH_VARARGS},
+    {"set_quantize", PyCMOR_set_quantize, METH_VARARGS},
     {"set_terminate_signal", PyCMOR_set_terminate_signal, METH_VARARGS},
     {"get_terminate_signal", PyCMOR_get_terminate_signal, METH_VARARGS},
     {NULL, NULL}                /*sentinel */

Src/cmor.c

@@ -9,6 +9,7 @@
 #include "cmor.h"
 #include "cmor_locale.h"
 #include <netcdf.h>
+#include <netcdf_filter.h>
 #include <udunits2.h>
 #include <time.h>
 #include <errno.h>
@@ -41,6 +42,20 @@ int nc_def_var_chunking(int i, int j, int k, size_t * l)
 };
 #endif
 
+#ifndef H5Z_FILTER_ZSTD
+int nc_def_var_zstandard(int i, int j, int k)
+{
+    return (0);
+};
+#endif
+
+#ifndef NC_QUANTIZE_BITGROOM
+int nc_def_var_quantize(int i, int j, int k, int l)
+{
+    return (0);
+};
+#endif
+
 /* -------------------------------------------------------------------- */
 /*      function declaration                                            */
 /* -------------------------------------------------------------------- */
@@ -720,6 +735,9 @@ void cmor_reset_variable(int var_id)
     cmor_vars[var_id].shuffle = 0;
     cmor_vars[var_id].deflate = 1;
     cmor_vars[var_id].deflate_level = 1;
+    cmor_vars[var_id].zstandard_level = 3;
+    cmor_vars[var_id].quantize_mode = 0;
+    cmor_vars[var_id].quantize_nsd = 1;
     cmor_vars[var_id].nomissing = 1;
     cmor_vars[var_id].iunits[0] = '\0';
     cmor_vars[var_id].ounits[0] = '\0';
@@ -1869,7 +1887,7 @@ int cmor_define_zfactors_vars(int var_id, int ncid, int *nc_dim,
     int ierr = 0, l, m, k, n, j, m2, found, nelts, *int_list = NULL;
     int dim_holder[CMOR_MAX_VARIABLES];
     int lnzfactors;
-    int ics, icd, icdl, ia;
+    int ics, icd, icdl, icz, icqm, icqn, ia;
     cmor_add_traceback("cmor_define_zfactors_vars");
     cmor_is_setup();
     lnzfactors = *nzfactors;
@@ -2066,8 +2084,24 @@ int cmor_define_zfactors_vars(int var_id, int ncid, int *nc_dim,
                         icd = cmor_tables[nTableID].vars[nTableID].deflate;
                         icdl =
                           cmor_tables[nTableID].vars[nTableID].deflate_level;
-                        ierr = nc_def_var_deflate(ncid, nc_zfactors[lnzfactors],
-                                                  ics, icd, icdl);
+                        icz =
+                          cmor_tables[nTableID].vars[nTableID].zstandard_level;
+                        icqm =
+                          cmor_tables[nTableID].vars[nTableID].quantize_mode;
+                        icqn =
+                          cmor_tables[nTableID].vars[nTableID].quantize_nsd;
+
+                        ierr = nc_def_var_quantize(ncid, nc_zfactors[lnzfactors],
+                                                icqm, icqn);
+                        if (icd != 0) {
+                            ierr |= nc_def_var_deflate(ncid, nc_zfactors[lnzfactors],
+                                                    ics, icd, icdl);
+                        } else {
+                            ierr |= nc_def_var_deflate(ncid, nc_zfactors[lnzfactors],
+                                                    ics, 0, 0);
+                            ierr |= nc_def_var_zstandard(ncid, nc_zfactors[lnzfactors],
+                                                    icz);
+                        }
 
                         if (ierr != NC_NOERR) {
                             snprintf(msg, CMOR_MAX_STRING,
@@ -3432,7 +3466,7 @@ void cmor_define_dimensions(int var_id, int ncid,
     int tmp_dims[2];
     int dims_bnds_ids[2];
     int nVarRefTblID = cmor_vars[var_id].ref_table_id;
-    int ics, icd, icdl;
+    int ics, icd, icdl, icz, icqm, icqn;
     int itmpmsg, itmp2, itmp3;
     int maxStrLen;
 
@@ -3784,9 +3818,21 @@ void cmor_define_dimensions(int var_id, int ncid,
                 ics = pVar->shuffle;
                 icd = pVar->deflate;
                 icdl = pVar->deflate_level;
-
-                ierr = nc_def_var_deflate(ncafid, nc_bnds_vars[i], ics, icd,
-                                          icdl);
+                icz = pVar->zstandard_level;
+                icqm = pVar->quantize_mode;
+                icqn = pVar->quantize_nsd;
+
+                ierr = nc_def_var_quantize(ncafid, nc_bnds_vars[i], icqm,
+                                        icqn);
+                if (icd != 0) {
+                    ierr |= nc_def_var_deflate(ncafid, nc_bnds_vars[i],
+                                            ics, icd, icdl);
+                } else {
+                    ierr |= nc_def_var_deflate(ncafid, nc_bnds_vars[i],
+                                            ics, 0, 0);
+                    ierr |= nc_def_var_zstandard(ncafid, nc_bnds_vars[i],
+                                            icz);
+                }
                 if (ierr != NC_NOERR) {
                     snprintf(msg, CMOR_MAX_STRING,
                              "NCError (%i: %s) defining compression\n! "
@@ -4039,7 +4085,7 @@ int cmor_grids_def(int var_id, int nGridID, int ncafid, int *nc_dim_af,
     int *int_list = NULL;
     char mtype;
     int nelts;
-    int ics, icd, icdl;
+    int ics, icd, icdl, icz, icqm, icqn;
 
     cmor_add_traceback("cmor_grids_def");
 /* -------------------------------------------------------------------- */
@@ -4306,9 +4352,29 @@ int cmor_grids_def(int var_id, int nGridID, int ncafid, int *nc_dim_af,
                       cmor_tables[cmor_vars[j].ref_table_id].vars[cmor_vars[j].
                                                                   ref_var_id].
                       deflate_level;
-
-                    ierr = nc_def_var_deflate(ncafid, nc_associated_vars[i],
-                                              ics, icd, icdl);
+                    icz =
+                      cmor_tables[cmor_vars[j].ref_table_id].vars[cmor_vars[j].
+                                                                  ref_var_id].
+                      zstandard_level;
+                    icqm =
+                      cmor_tables[cmor_vars[j].ref_table_id].vars[cmor_vars[j].
+                                                                  ref_var_id].
+                      quantize_mode;
+                    icqn =
+                      cmor_tables[cmor_vars[j].ref_table_id].vars[cmor_vars[j].
+                                                                  ref_var_id].
+                      quantize_nsd;
+                    ierr = nc_def_var_quantize(ncafid, nc_associated_vars[i],
+                                            icqm, icqn);
+                    if (icd != 0) {
+                        ierr |= nc_def_var_deflate(ncafid, nc_associated_vars[i],
+                                                ics, icd, icdl);
+                    } else {
+                        ierr |= nc_def_var_deflate(ncafid, nc_associated_vars[i],
+                                                ics, 0, 0);
+                        ierr |= nc_def_var_zstandard(ncafid, nc_associated_vars[i],
+                                                icz);
+                    }
                     if (ierr != NC_NOERR) {
                         snprintf(msg, CMOR_MAX_STRING,
                                  "NetCDF Error (%i: %s) defining\n! "
@@ -5093,7 +5159,7 @@ void cmor_create_var_attributes(int var_id, int ncid, int ncafid,
     int nVarRefTblID = cmor_vars[var_id].ref_table_id;
     int nelts;
     int *int_list = NULL;
-    int ics, icd, icdl;
+    int ics, icd, icdl, icz, icqm, icqn;
     int bChunk;
     cmor_add_traceback("cmor_create_var_attributes");
 /* -------------------------------------------------------------------- */
@@ -5167,7 +5233,18 @@ void cmor_create_var_attributes(int var_id, int ncid, int ncafid,
         ics = pVar->shuffle;
         icd = pVar->deflate;
         icdl = pVar->deflate_level;
-        ierr = nc_def_var_deflate(ncid, pVar->nc_var_id, ics, icd, icdl);
+        icz = pVar->zstandard_level;
+        icqm = pVar->quantize_mode;
+        icqn = pVar->quantize_nsd;
+        ierr = nc_def_var_quantize(ncid, pVar->nc_var_id, icqm, icqn);
+
+        // Only use zstandard compression if deflate is disabled
+        if (icd != 0) {
+            ierr |= nc_def_var_deflate(ncid, pVar->nc_var_id, ics, icd, icdl);
+        } else {
+            ierr |= nc_def_var_deflate(ncid, pVar->nc_var_id, ics, 0, 0);
+            ierr |= nc_def_var_zstandard(ncid, pVar->nc_var_id, icz);
+        }
 
         if (ierr != NC_NOERR) {
             snprintf(msg, CMOR_MAX_STRING,

Src/cmor_CV.c

@@ -2539,6 +2539,9 @@ int cmor_CV_variable(int *var_id, char *name, char *units,
     cmor_vars[vrid].shuffle = refvar.shuffle;
     cmor_vars[vrid].deflate = refvar.deflate;
     cmor_vars[vrid].deflate_level = refvar.deflate_level;
+    cmor_vars[vrid].zstandard_level = refvar.zstandard_level;
+    cmor_vars[vrid].quantize_mode = refvar.quantize_mode;
+    cmor_vars[vrid].quantize_nsd = refvar.quantize_nsd;
     cmor_vars[vrid].first_bound = startimebnds;
     cmor_vars[vrid].last_bound = endtimebnds;
     cmor_vars[vrid].first_time = startime;

Src/cmor_cfortran_interface.c

@@ -92,6 +92,23 @@ int cmor_set_deflate_cff_(int *var_id, int *shuffle,
     return (cmor_set_deflate(*var_id, *shuffle, *deflate, *deflate_level));
 }
 
+/************************************************************************/
+/*                       cmor_set_zstandard_cff_()                        */
+/************************************************************************/
+int cmor_set_zstandard_cff_(int *var_id, int *zstandard_level)
+{
+    return (cmor_set_zstandard(*var_id, *zstandard_level));
+}
+
+/************************************************************************/
+/*                       cmor_set_quantize_cff_()                        */
+/************************************************************************/
+int cmor_set_quantize_cff_(int *var_id, int *quantize_mode,
+                          int *quantize_level)
+{
+    return (cmor_set_quantize(*var_id, *quantize_mode, *quantize_level));
+}
+
 /************************************************************************/
 /*                  cmor_get_variable_attribute_cff_()                  */
 /************************************************************************/