Add or modify Makefile recipes to enable regeneration of *.rdf files and retain expected *.ttl files content

.github/workflows/unit-tests.yml

@@ -20,7 +20,7 @@ jobs:
         with:
           fetch-depth: 0  # Shallow clones should be disabled for a better relevancy of analysis
       - name: Build and Run Tests
-        run: mvn clean install
+        run: make install unit-tests
       - name: current dir
         run: pwd
       - name: current dir

.gitignore

@@ -325,3 +325,4 @@ lib64
 pyvenv.cfg
 /build/
 /node_modules/
+.temp

Makefile

@@ -1,6 +1,7 @@
 
 # Model2owl directory
 MODEL2OWL_FOLDER?=.
+ABSOLUTE_MODEL2OWL_FOLDER?=$(shell realpath "${MODEL2OWL_FOLDER}")
 # rdflib version
 RDF_LIB_VERSION?=6.2.0
 #Saxon path
@@ -33,6 +34,12 @@ TURTLE_FILELIST=$(shell ls ${ONTOLOGY_FOLDER_PATH}/*.ttl)
 # Widoco variables
 WIDOCO_RDF_INPUT_FILE_PATH?=test/reasoning-investigation/model-2020-12-16/ePO_restrictions.rdf
 WIDOCO_OUTPUT_FOLDER_PATH?=output/widoco
+NAMESPACES_USER_XML_FILE_PATH?=${MODEL2OWL_FOLDER}/test/ePO-default-config/namespaces.xml
+INTERM_FOLDER_PATH?=${ABSOLUTE_MODEL2OWL_FOLDER}/.temp
+ENRICHED_NAMESPACES_XML_PATH:=${INTERM_FOLDER_PATH}/enriched-namespaces.xml
+NAMESPACES_AS_RDFPIPE_ARGS=$(shell ${MODEL2OWL_FOLDER}/scripts/get_namespaces.sh ${ENRICHED_NAMESPACES_XML_PATH})
+RDF_XML_MIME_TYPE:='application/rdf+xml'
+TURTLE_MIME_TYPE:='turtle'
 
 # download saxon library 	
 get-saxon:
@@ -61,12 +68,16 @@ get-widoco:
 ######################################################################################
 # Download, install saxon, xspec, rdflib and other dependencies
 ######################################################################################
-install:  get-saxon get-rdflib get-widoco
+install:  get-saxon create-virtual-env get-rdflib get-widoco
 
 ############################ Main tasks ##############################################
 # Run unit_tests
 unit-tests:
-	@mvn install
+	@make test-prerequisites
+	@mvn install -Dsaxon.options.enrichedNamespacesPath=${ENRICHED_NAMESPACES_XML_PATH}
+
+test-prerequisites:
+	@make gen-enriched-ns-file
 
 create-virtual-env:
 	@python -m venv model2owl-venv
@@ -77,51 +88,95 @@ create-virtual-env:
 # make generate-glossary XMI_INPUT_FILE_PATH=/home/mypc/work/model2owl/eNotice_CM.xml OUTPUT_GLOSSARY_PATH=/home/mypc/work/model2owl/glossary
 generate-glossary:
 	@mkdir -p "${OUTPUT_GLOSSARY_PATH}"
+	@make gen-enriched-ns-file
 	@echo Input file path: ${XMI_INPUT_FILE_PATH}
 	@echo Input file name: ${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}
 	@cp -rf ./src/static "${OUTPUT_GLOSSARY_PATH}"
-	@java -jar ${SAXON} -s:${XMI_INPUT_FILE_PATH} -xsl:${MODEL2OWL_FOLDER}/src/html-model-glossary.xsl -o:${OUTPUT_GLOSSARY_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_glossary.html
+	@java -jar ${SAXON} -s:${XMI_INPUT_FILE_PATH} \
+		-xsl:${MODEL2OWL_FOLDER}/src/html-model-glossary.xsl \
+		-o:${OUTPUT_GLOSSARY_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_glossary.html \
+		enrichedNamespacesPath="${ENRICHED_NAMESPACES_XML_PATH}"
 	@echo The glossary is located at the following location:
 	@echo
 	@ls -lh ${OUTPUT_GLOSSARY_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_glossary.html
 	@echo
 
 generate-convention-report:
 	@mkdir -p "${OUTPUT_CONVENTION_REPORT_PATH}"
+	@make gen-enriched-ns-file
 	@echo Input file path: ${XMI_INPUT_FILE_PATH}
 	@echo Input file name: ${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}
 	@cp -rf ./src/static "${OUTPUT_CONVENTION_REPORT_PATH}"
-	@java -jar ${SAXON} -s:${XMI_INPUT_FILE_PATH} -xsl:${MODEL2OWL_FOLDER}/src/html-conventions-report.xsl -o:${OUTPUT_CONVENTION_REPORT_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_convention_report.html
+	@java -jar ${SAXON} -s:${XMI_INPUT_FILE_PATH} \
+		-xsl:${MODEL2OWL_FOLDER}/src/html-conventions-report.xsl \
+		-o:${OUTPUT_CONVENTION_REPORT_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_convention_report.html \
+		enrichedNamespacesPath="${ENRICHED_NAMESPACES_XML_PATH}"
 	@echo The convention report is located at the following location:
 	@echo
 	@ls -lh ${OUTPUT_CONVENTION_REPORT_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_convention_report.html
 	@echo
 
 generate-convention-SVRL-report:
 	@mkdir -p "${OUTPUT_CONVENTION_REPORT_PATH}"
+	@make gen-enriched-ns-file
 	@echo Input file path: ${XMI_INPUT_FILE_PATH}
 	@echo Input file name: ${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}
 	@cp -rf ./src/static "${OUTPUT_CONVENTION_REPORT_PATH}"
-	@java -jar ${SAXON} -s:${XMI_INPUT_FILE_PATH} -xsl:${MODEL2OWL_FOLDER}/src/svrl-conventions-report.xsl -o:${OUTPUT_CONVENTION_REPORT_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_convention_svrl_report.xml
+	@java -jar ${SAXON} -s:${XMI_INPUT_FILE_PATH} \
+		-xsl:${MODEL2OWL_FOLDER}/src/svrl-conventions-report.xsl \
+		-o:${OUTPUT_CONVENTION_REPORT_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_convention_svrl_report.xml \
+		enrichedNamespacesPath="${ENRICHED_NAMESPACES_XML_PATH}"
 	@echo The convention report is located at the following location:
 	@echo
 	@ls -lh ${OUTPUT_CONVENTION_REPORT_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_convention_svrl_report.xml
 	@echo
 #Example how to run transformation commands :
 # make owl-core XMI_INPUT_FILE_PATH=/home/mypc/work/model2owl/eNotice_CM.xml OUTPUT_FOLDER_PATH=./my-folder
 owl-core:
-	@java -jar ${SAXON} -s:${XMI_INPUT_FILE_PATH} -xsl:${MODEL2OWL_FOLDER}/src/owl-core.xsl -o:${OUTPUT_FOLDER_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}.rdf
+	@make gen-enriched-ns-file
+	@java -jar ${SAXON} -s:${XMI_INPUT_FILE_PATH} -xsl:${MODEL2OWL_FOLDER}/src/owl-core.xsl \
+		-o:${OUTPUT_FOLDER_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}.tmp.rdf \
+		enrichedNamespacesPath="${ENRICHED_NAMESPACES_XML_PATH}"
+	@make convert-between-serialization-formats INPUT_FORMAT=${RDF_XML_MIME_TYPE} \
+		OUTPUT_FORMAT=${RDF_XML_MIME_TYPE} \
+		FILE_PATH=${OUTPUT_FOLDER_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}.tmp.rdf \
+		OUTPUT_FILE_PATH=${OUTPUT_FOLDER_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}.rdf
 	@echo Output owl core file:
 	@ls -lh ${OUTPUT_FOLDER_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}.rdf
+	@rm -f ${OUTPUT_FOLDER_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}.tmp.rdf
+
 owl-restrictions:
-	@java -jar ${SAXON} -s:${XMI_INPUT_FILE_PATH} -xsl:${MODEL2OWL_FOLDER}/src/owl-restrictions.xsl -o:${OUTPUT_FOLDER_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_restrictions.rdf
+	@make gen-enriched-ns-file
+	@java -jar ${SAXON} -s:${XMI_INPUT_FILE_PATH} -xsl:${MODEL2OWL_FOLDER}/src/owl-restrictions.xsl \
+		-o:${OUTPUT_FOLDER_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_restrictions.tmp.rdf \
+		enrichedNamespacesPath="${ENRICHED_NAMESPACES_XML_PATH}"
+	@make convert-between-serialization-formats INPUT_FORMAT=${RDF_XML_MIME_TYPE} \
+		OUTPUT_FORMAT=${RDF_XML_MIME_TYPE} \
+		FILE_PATH=${OUTPUT_FOLDER_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_restrictions.tmp.rdf \
+		OUTPUT_FILE_PATH=${OUTPUT_FOLDER_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_restrictions.rdf
 	@echo Output owl restrictions file:
 	@ls -lh ${OUTPUT_FOLDER_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_restrictions.rdf
+	@rm -f ${OUTPUT_FOLDER_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_restrictions.tmp.rdf
+
 shacl:
-	@java -jar ${SAXON} -s:${XMI_INPUT_FILE_PATH} -xsl:${MODEL2OWL_FOLDER}/src/shacl-shapes.xsl -o:${OUTPUT_FOLDER_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_shapes.rdf
+	@make gen-enriched-ns-file
+	@java -jar ${SAXON} -s:${XMI_INPUT_FILE_PATH} -xsl:${MODEL2OWL_FOLDER}/src/shacl-shapes.xsl \
+		-o:${OUTPUT_FOLDER_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_shapes.tmp.rdf \
+		enrichedNamespacesPath="${ENRICHED_NAMESPACES_XML_PATH}"
+	@make convert-between-serialization-formats INPUT_FORMAT=${RDF_XML_MIME_TYPE} \
+		OUTPUT_FORMAT=${RDF_XML_MIME_TYPE} \
+		FILE_PATH=${OUTPUT_FOLDER_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_shapes.tmp.rdf \
+		OUTPUT_FILE_PATH=${OUTPUT_FOLDER_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_shapes.rdf
 	@echo Output shacl file location:
 	@ls -lh ${OUTPUT_FOLDER_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_shapes.rdf
+	@rm -f ${OUTPUT_FOLDER_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_shapes.tmp.rdf
 
+# Generate enriched namespaces XML file which contains user namespaces (defined
+# in namespaces.xml) and internal namespaces (such as core-shape)
+gen-enriched-ns-file:
+	@mkdir -p ${INTERM_FOLDER_PATH}
+	@java -jar ${SAXON} -s:${NAMESPACES_USER_XML_FILE_PATH} -xsl:${MODEL2OWL_FOLDER}/src/xml/enriched-namespaces.xsl \
+		-o:${ENRICHED_NAMESPACES_XML_PATH}
 
 # Combine xmi UML files
 # all files for combine should be in test/test-multi-xmi (or in XMI_MERGED_OUTPUT_FOLDER_PATH)
@@ -141,10 +196,16 @@ merge-xmi:
 # make convert-to-turtle ONTOLOGY_FOLDER_PATH=./my-folder
 # ONTOLOGY_FOLDER_PATH is the the path to the folder containing .rdf files for converting to turtle or .ttl files to convert to rdf
 convert-rdf-to-turtle:
+	@make gen-enriched-ns-file
 	@for FILE_PATH in ${RDF_FILELIST}; do \
 		echo Converting $${FILE_PATH} into Turtle; \
 		source model2owl-venv/bin/activate; \
-		rdfpipe -i application/rdf+xml -o  turtle $${FILE_PATH} > $${FILE_PATH%.*}.ttl; \
+		make convert-between-serialization-formats \
+			INPUT_FORMAT=${RDF_XML_MIME_TYPE} \
+			OUTPUT_FORMAT=${TURTLE_MIME_TYPE}  \
+			FILE_PATH=$${FILE_PATH}  \
+			OUTPUT_FILE_PATH=$${FILE_PATH%.*}.ttl \
+			USE_NAMESPACES=1; \
 		echo Input in RDF/XML format;  \
 		echo $${FILE_PATH};  \
 		echo " ==> Output in Turtle format";  \
@@ -182,6 +243,36 @@ convert-rdf-to-rdf:
 		echo " ==> Output in RDF/XML format";  \
 		ls -lh $${FILE_PATH%.*}.rdf;  \
 	done
+
+# A generic recipe for converting RDF data from one serialization format to 
+# another. It can also be used to regenerate a file using the same format.
+# 
+# Arguments:
+# 	FILE_PATH: Input RDF file in any allowed serialization format
+# 	OUTPUT_FILE_PATH: Path for the output file
+# 	INPUT_FORMAT: a MIME type of the given input RDF file
+# 	OUTPUT_FORMAT: a MIME type of any of the valid RDF serializations
+#   USE_NAMESPACES: optional; if non-empty then namespaces (from the
+#                   enriched-namespaces.xml file). This can be used if the input
+#                   (FILE_PATH) doesn't include namespaces we want to be applied
+#                   (e.g. to have compact instead of full URIs in the output
+#                   file).
+#					
+# Supported MIME types: https://rdflib.readthedocs.io/en/7.0.0/plugin_serializers.html
+# 	
+# Example:
+# make convert-between-serialization-formats
+#   INPUT_FORMAT='application/rdf+xml'
+#   OUTPUT_FORMAT='application/rdf+xml' 
+#   FILE_PATH=output/ePO_core.tmp.rdf 
+#   OUTPUT_FILE_PATH=output/ePO_core.rdf
+# 	USE_NAMESPACES=1
+convert-between-serialization-formats:
+	@source model2owl-venv/bin/activate; \
+	rdfpipe -i ${INPUT_FORMAT} -o ${OUTPUT_FORMAT} \
+		$(if $(USE_NAMESPACES),${NAMESPACES_AS_RDFPIPE_ARGS}) \
+		${FILE_PATH} > ${OUTPUT_FILE_PATH}
+
 # make validate-rdf-file FILE_TO_VALIDATE_PATH=./output/eFulfilment.rdf
 validate-rdf-file:
 	@$(JENA_RIOT_TOOL) --validate $(FILE_TO_VALIDATE_PATH)

pom.xml

@@ -25,6 +25,7 @@
                     <generateSurefireReport>true</generateSurefireReport>
                     <xspecProperties>
                         <xspec.fail>false</xspec.fail>
+                        <saxon.custom.options>enrichedNamespacesPath=${saxon.options.enrichedNamespacesPath}</saxon.custom.options>
                     </xspecProperties>
                 </configuration>
 <!--                <dependencies>-->

scripts/get_namespaces.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+# 
+# Gets namespaces from an XML file with the model2owl project namespaces and
+# prepares argument list from them to be used with `rdfpipe` tool. Uses Saxon
+# installed in the project main directory.
+# 
+# USAGE: get_namespaces.sh NAMESPACES_XML_FILE_PATH
+PROJECT_DIR=$(dirname $(dirname $(realpath ${BASH_SOURCE[0]})))
+SAXON=${PROJECT_DIR}/saxon/saxon.jar
+
+if [ -z "$1" ]; then
+	echo "ERROR: path to *.xml file with namespaces not given."
+	exit 1
+fi
+namespaces_file_path="$1"
+
+namespaces_file_dir=$(dirname $(realpath $namespaces_file_path))
+namespaces_file_name=$(basename $namespaces_file_path)
+
+cd ${namespaces_file_dir}
+namespaces=$(
+    java -cp $SAXON net.sf.saxon.Query -s:${namespaces_file_name} \
+		-qs:'for $x in /*:prefixes/*:prefix return concat(string($x/@name), "=", string($x/@value))' \
+		\!method=text
+)
+ns_args=$( \
+	echo "$namespaces" | tr ' ' '\n' | awk '{printf("--ns='\''%s'\'' ", $0)}'
+)
+
+echo "$ns_args"

src/common/checkers.xsl

@@ -512,15 +512,15 @@
     </xsl:function>
 
     <xd:doc>
-        <xd:desc> This function will check if a given list of namespaces are defined in
-            namespaces.xml file. If not all the namespaces were defined it will return a list with
-            those namespaces</xd:desc>
+        <xd:desc> This function will check if a given list of namespaces are
+        defined in enriched-namespaces.xml file. If not all the namespaces were
+        defined it will return a list with those namespaces</xd:desc>
         <xd:param name="listOfNamespaces"/>
     </xd:doc>
     <xsl:function name="f:isAllNamespacesDefined">
         <xsl:param name="listOfNamespaces"/>
         <xsl:variable name="definedNamespaces"
-            select="($namespacePrefixes/*:prefixes/*:prefix/@name)"/>
+            select="($internalNamespacePrefixes/*:prefixes/*:prefix/@name)"/>
         <xsl:variable name="listOfNotDefinedNamespaces"
             select="functx:value-except($listOfNamespaces, $definedNamespaces)"/>
         <xsl:sequence

src/common/utils.xsl

@@ -21,6 +21,13 @@
     <xsl:import href="fetchers.xsl"/>
     <xsl:import href="functx-1.0.1-doc.xsl"/>
 
+    <xsl:param name="enrichedNamespacesPath"/>
+    <xsl:variable name="internalNamespacePrefixes" select="
+        if (boolean($enrichedNamespacesPath)) then
+            fn:doc($enrichedNamespacesPath)
+        else fn:error(xs:QName('missing-parameter'), 'enrichedNamespacesPath is not given.')
+    "/>
+
     <xd:doc>
         <xd:desc> Lookup a data-type in the xsd and rdf accepted data-type document (usually an
             external file with xsd and rdf data-types definitions) and return false or the data-type
@@ -61,7 +68,7 @@
     <xsl:function name="f:getNamespaceURI" as="xs:string">
         <xsl:param name="prefix"/>
 
-        <xsl:variable name="fetch" select="f:getNamespaceValues($prefix, $namespacePrefixes)"/>
+        <xsl:variable name="fetch" select="f:getNamespaceValues($prefix, $internalNamespacePrefixes)"/>
         <xsl:sequence
             select="
                 if (boolean($fetch)) then
@@ -492,4 +499,13 @@
         />
     </xsl:function>
 
+    <xd:doc>
+        <xd:desc>This template declares set of namespaces to be defined in top element of an output file</xd:desc>
+    </xd:doc>
+    <xsl:template name="namespacesDeclaration">
+        <xsl:for-each select="$internalNamespacePrefixes/*:prefixes/*:prefix">              
+            <xsl:namespace name="{./@name}" select="./@value"/>
+        </xsl:for-each>
+    </xsl:template>
+
 </xsl:stylesheet>

src/owl-core.xsl

@@ -46,9 +46,7 @@
     </xd:doc>
     <xsl:template match="/">
         <rdf:RDF>
-            <xsl:for-each select="$namespacePrefixes/*:prefixes/*:prefix">              
-                <xsl:namespace name="{./@name}" select="./@value"/>
-            </xsl:for-each>   
+            <xsl:call-template name="namespacesDeclaration"/>
             <xsl:call-template name="ontology-header"/>
             <xsl:apply-templates/>
             <xsl:call-template name="generalisationsWithDistinctTargetsInCoreLayer"/>
@@ -63,7 +61,7 @@
     <xsl:template name="ontology-header">
 
         <owl:Ontology rdf:about="{$coreArtefactURI}">         
-            <xsl:for-each select="$namespacePrefixes/*:prefixes/*:prefix/@importURI">              
+            <xsl:for-each select="$internalNamespacePrefixes/*:prefixes/*:prefix/@importURI">              
                 <owl:imports rdf:resource="{.}"/>
             </xsl:for-each>      
 

src/owl-restrictions.xsl

@@ -40,11 +40,7 @@
     </xd:doc>
     <xsl:template match="/">
         <rdf:RDF>
-            <xsl:for-each select="$namespacePrefixes/*:prefixes/*:prefix">              
-                <xsl:namespace name="{./@name}" select="./@value"/>
-            </xsl:for-each>
-            <xsl:namespace name="{fn:concat($moduleReference, '-res')}" select="fn:concat($base-restriction-uri,$defaultDelimiter)"/>
-
+            <xsl:call-template name="namespacesDeclaration"/>            
             <xsl:call-template name="ontology-header"/>
             <xsl:apply-templates/>
             <xsl:call-template name="generalisationsWithDistinctTargetsInReasoningLayer"/>
@@ -58,7 +54,7 @@
     </xd:doc>
     <xsl:template name="ontology-header">
         <owl:Ontology rdf:about="{$restrictionsArtefactURI}">            
-            <xsl:for-each select="$namespacePrefixes/*:prefixes/*:prefix/@importURI">              
+            <xsl:for-each select="$internalNamespacePrefixes/*:prefixes/*:prefix/@importURI">              
                 <owl:imports rdf:resource="{.}"/>
             </xsl:for-each>      
             <owl:imports rdf:resource="{$coreArtefactURI}"/>

src/shacl-shapes.xsl

@@ -44,13 +44,7 @@
     </xd:doc>
     <xsl:template match="/">
         <rdf:RDF>
-            <xsl:for-each select="$namespacePrefixes/*:prefixes/*:prefix">              
-                <xsl:namespace name="{./@name}" select="./@value"/>
-            </xsl:for-each>
-            <xsl:namespace name="{fn:concat($moduleReference, '-res')}" select="fn:concat($base-restriction-uri,$defaultDelimiter)"/>
-            <xsl:namespace name="{fn:concat($moduleReference, '-shape')}" select="fn:concat($base-shape-uri,$defaultDelimiter)"/>
-
-
+            <xsl:call-template name="namespacesDeclaration"/>
             <xsl:call-template name="ontology-header"/>
             <xsl:apply-templates/>
         </rdf:RDF>
@@ -62,7 +56,7 @@
     <xsl:template name="ontology-header">
         <owl:Ontology rdf:about="{$shapeArtefactURI}">
 
-            <xsl:for-each select="$namespacePrefixes/*:prefixes/*:prefix/@importURI">              
+            <xsl:for-each select="$internalNamespacePrefixes/*:prefixes/*:prefix/@importURI">              
                 <owl:imports rdf:resource="{.}"/>
             </xsl:for-each>      
             <owl:imports rdf:resource="{$coreArtefactURI}"/>