diff --git a/scripts/coverage/coverage_score.sh b/scripts/coverage/coverage_score.sh new file mode 100755 index 0000000..6558bf7 --- /dev/null +++ b/scripts/coverage/coverage_score.sh @@ -0,0 +1,105 @@ +#!/bin/bash + +# Processes code coverage reports generated by Xspec in order to calculate +# coverage score (per module and overall). The score is based on a number of +# covered / not covered functions (xsl:function) and templates (xsl:template). +# This script provides the feature which is missed in Xspec (as Xspec +# coverage reports only highlights parts of code not covered with tests). +# In addition, the script stores lists of covered and not covered functions +# and templates in /test/unitTests/coverage/coverage-details. +# Expects coverage reports (*-coverage.html files) stored in /test/unitTests +# +# Tested on the following setup: +# Xspec: v3.0.3 +# Saxon: v12.5 + +SCRIPT_DIR=$(realpath $(dirname "${BASH_SOURCE[0]}")) +PROJ_DIR=$(dirname $(dirname $SCRIPT_DIR)) + +COV_REPORTS_DIR=${PROJ_DIR}/test/unitTests +COV_REPORTS_OUTPUT_DIR=${PROJ_DIR}/test/unitTests/coverage +COV_REPORTS_DETAILS_OUTPUT_DIR=${PROJ_DIR}/test/unitTests/coverage/coverage-details +MISSED_CSS_CLS_HTML_CHUNK='class="missed"' +HIT_CSS_CLS_HTML_CHUNK='class="hit"' +XSL_FUNC_HTML_CHUNK='xsl:function name="f:' +XSL_TMPL_HTML_CHUNK='xsl:template' +OVERALL_SCORE_FILE=${COV_REPORTS_OUTPUT_DIR}/coverage-summary.txt +OVERALL_SCORE_FILE_TSV=${COV_REPORTS_OUTPUT_DIR}/coverage-summary.tsv + +rm -rf "$COV_REPORTS_OUTPUT_DIR" +mkdir -p ${COV_REPORTS_DETAILS_OUTPUT_DIR} + +hits_total=0 +missed_total=0 + +shopt -s globstar +cd "$COV_REPORTS_DIR" +for report in **/*-coverage.html; do + report_slug=$(echo "$report" | sed 's|/|_|g' | cut -f1 -d'.') + xsl_module=$( \ + head $report \ + | grep "Test Coverage Report for" \ + | awk -F '/src/' '{printf("src/%s\n", $2)}' \ + | cut -f1 -d'<' \ + | cut -f1 -d' ' + ) + hits_file="${COV_REPORTS_DETAILS_OUTPUT_DIR}/${report_slug}-hits.txt" + missed_file="${COV_REPORTS_DETAILS_OUTPUT_DIR}/${report_slug}-missed.txt" + + grep "$MISSED_CSS_CLS_HTML_CHUNK" $report \ + | grep "$XSL_FUNC_HTML_CHUNK" \ + | awk -F'xsl:function name="' '{print $2}' \ + | cut -d'"' -f1 > "$missed_file" + + grep "$MISSED_CSS_CLS_HTML_CHUNK" $report \ + | grep "$XSL_TMPL_HTML_CHUNK" \ + | grep -v '</xsl:template>' \ + | sed -e 's|<|$|g' -e 's|>|$|g' \ + | cut -f2 -d'$' >> "$missed_file" + + grep "$HIT_CSS_CLS_HTML_CHUNK" $report \ + | grep "$XSL_FUNC_HTML_CHUNK" \ + | awk -F'xsl:function name="' '{print $2}' \ + | cut -d'"' -f1 > "$hits_file" + + grep "$HIT_CSS_CLS_HTML_CHUNK" $report \ + | grep "$XSL_TMPL_HTML_CHUNK" \ + | grep -v '</xsl:template>' \ + | sed -e 's|<|$|g' -e 's|>|$|g' \ + | cut -f2 -d'$' >> "$hits_file" + + hits_cnt=$(wc -l "$hits_file" | awk '{print $1}') + missed_cnt=$(wc -l "$missed_file" | awk '{print $1}') + if (( (hits_cnt + missed_cnt) > 0 )); then + hits_total=$((hits_total + hits_cnt)) + missed_total=$((missed_total + missed_cnt)) + coverage=$( + awk \ + -v hits=$hits_cnt \ + -v missed=$missed_cnt \ + 'BEGIN {printf("%.2f\n", hits/(hits+missed))}'\ + ) + echo "$xsl_module ($report_slug): $coverage" >> $OVERALL_SCORE_FILE + echo -e "$xsl_module ($report_slug)\t$coverage" >> $OVERALL_SCORE_FILE_TSV + else + echo "WARN: Skipping $report results" + fi +done + +echo -n "OVERALL: " >> $OVERALL_SCORE_FILE +echo -ne "OVERALL\t" >> $OVERALL_SCORE_FILE_TSV +overall_coverage=$( + awk \ + -v hits=$hits_total \ + -v missed=$missed_total \ + 'BEGIN {printf("%.2f\n", hits/(hits+missed))}'\ +) +echo " $overall_coverage ($hits_total / $((hits_total + missed_total)))" >> $OVERALL_SCORE_FILE +echo "$overall_coverage" >> $OVERALL_SCORE_FILE_TSV + +cd - &> /dev/null + +echo "Code coverage for xsl:function (f:) and xsl:template" +echo "Overall coverage score: $overall_coverage" +echo "Coverage summary report files generated: $OVERALL_SCORE_FILE, $OVERALL_SCORE_FILE_TSV" +echo "Coverage details stored in $COV_REPORTS_DETAILS_OUTPUT_DIR" diff --git a/scripts/coverage/gen_reports.sh b/scripts/coverage/gen_reports.sh new file mode 100755 index 0000000..e8aa5b9 --- /dev/null +++ b/scripts/coverage/gen_reports.sh @@ -0,0 +1,14 @@ +#!/bin/bash +SCRIPT_DIR=$(realpath $(dirname "${BASH_SOURCE[0]}")) +PROJ_DIR=$(dirname $(dirname $SCRIPT_DIR)) + +SAXON_DIR=${PROJ_DIR}/saxon-12.5 +XSPEC=${PROJ_DIR}/target/xspecmaven/xspec-framework/xspec-3.0.3/bin/xspec.sh + +SAXON_HOME=${SAXON_DIR} find ${PROJ_DIR}/test/unitTests -name '*.xspec' -exec sh -c "$XSPEC -c {}" \; + +# uncomment to copy the reports into a single new directory +# cd ${PROJ_DIR}/test/unitTests +# mkdir -p coverage/coverage-reports +# find . -name '*-coverage.html' -exec cp -t coverage/coverage-reports {} + +# cd - &> /dev/null diff --git a/scripts/coverage/lib/xspec-3.0.3/src/reporter/coverage-report.xsl b/scripts/coverage/lib/xspec-3.0.3/src/reporter/coverage-report.xsl new file mode 100644 index 0000000..a68b07d --- /dev/null +++ b/scripts/coverage/lib/xspec-3.0.3/src/reporter/coverage-report.xsl @@ -0,0 +1,597 @@ + + + + + + + + + + + + + + + + + + + http://www.jenitennison.com/xslt/xspec/coverage-report.xsl + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <xsl:text expand-text="yes">Test Coverage Report for {fmt:format-uri($stylesheet-uri)}</xsl:text> + + + + + + + +

Test Coverage Report

+

+ Stylesheet: + + + +

+ + + + + + + + + + + + + + + + +

+ module: {fmt:format-uri($stylesheet-uri)}; {$number-of-lines} lines +

+ + +

not used

+ + + 
+               
+               : 
+               
+                  
+                  
+                  
+               
+
+ + + + + + + \s+ + (?:[^>\s]+) + \s* + = + \s* + (?: + "(?:[^"]*)" + | + '(?:[^']*)' + ) + + + + + + + + ( + ([^<]+) + | + (<!-- + (?:[^-]|-[^-])* + -->) + | + (<\? + (?:[^?]|\?[^>])* + \?>) + | + (<!\[CDATA\[ + (?:[^\]]|\][^\]]|\]\][^>])* + \]\]>) + | + (</ + ([^>]+) + >) + | + (< + ([^>/\s]+) + (?: + (?: + + )* + ) + \s* + (/?) + > + ) + ) + + + + + + + + + + + + + + + + + + + + + + + + + + + + ERROR: unmatched string: {.} + + + + + + + + + + + + + + + + + + + + + + {format-number($line-number + position(), $number-format)}: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ignored + + ignored + + + + + + + + hit + missed + + + + + + + + + hit + ignored + + + + + + + ignored + missed + + ignored + + + + + + + + missed + + + + ignored + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ERROR: more than one coverage identified for: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +