Enabled VSS4 support for Seat Service, fixed integration tests and DevContainer

.dapr/config.yaml

@@ -15,6 +15,7 @@ apiVersion: dapr.io/v1alpha1
 kind: Configuration
 metadata:
   name: config
+  namespace: kuksa
 spec:
   # tracing:
   #   samplingRate: "15"

.devcontainer/Dockerfile

@@ -12,7 +12,8 @@
 # ********************************************************************************/
 
 # Download base image of Ubuntu
-FROM kuksa.val.services/ci:latest
+# NOTE: it may be required to build the image locally (check root README.md)
+FROM ghcr.io/eclipse/kuksa.val.services/oci_kuksa-val-services-ci:main
 
 # If you get proxy errors, make sure Docker is configured correctly.
 # Hint: https://dev.to/zyfa/setup-the-proxy-for-dockerfile-building--4jc8
@@ -22,14 +23,20 @@ USER root
 # To prevent interactive shells
 ENV DEBIAN_FRONTEND=noninteractive
 
+### TODO: Move all installs from here to base image to speedup codespaces starutp
+
 # Install basic utils needed inside devcontainer
 RUN apt-get update && \
-  apt-get install -qqy curl wget zip  \
-  git \
+  apt-get install -qqy wget zip  \
   bash \
   xz-utils \
   apt-transport-https \
-  jq
+  file
+
+# python3.9 is required for feedercan vscode task
+RUN apt-get install -qqy python3.9
+# hack to install proper version of pip3.9
+RUN curl https://bootstrap.pypa.io/get-pip.py | python3.9
 
 # Set the locale
 RUN apt-get install -qqy locales locales-all
@@ -40,7 +47,7 @@ ENV LANGUAGE en_US.UTF-8
 # Set timezone inside Docker container
 ENV TZ=UTC
 RUN echo $TZ > /etc/timezone && \
-  apt-get update && apt-get install -qqy tzdata && \
+  apt-get install -qqy tzdata && \
   rm /etc/localtime && \
   dpkg-reconfigure -f noninteractive tzdata
 

.devcontainer/devcontainer.json

@@ -4,21 +4,23 @@
     "dockerfile": "Dockerfile"
   },
   "runArgs": ["--init", "--privileged"],
-  "settings": {
-    "terminal.integrated.defaultProfile.linux": "bash",
-    "http.proxyStrictSSL": false
+  "customizations": {
+    "vscode": {
+      "settings": {
+        "terminal.integrated.defaultProfile.linux": "bash",
+        "http.proxyStrictSSL": false
+      },
+      "extensions": [
+        "yzhang.markdown-all-in-one",
+        "arturock.gitstash",
+        "eamodio.gitlens",
+        "streetsidesoftware.code-spell-checker",
+        "twxs.cmake"
+      ]
+    }
   },
   "features": {
-    "docker-in-docker": {
-      "version": "latest",
-      "moby": true
+    "ghcr.io/devcontainers/features/docker-in-docker:2": {
     }
-  },
-  "extensions": [
-    "yzhang.markdown-all-in-one",
-    "arturock.gitstash",
-    "eamodio.gitlens",
-    "streetsidesoftware.code-spell-checker",
-    "twxs.cmake"
-  ]
+  }
 }

.github/workflows/integration_test.yml

@@ -74,7 +74,7 @@ jobs:
           # This is needed if the default VSS version used by Databroker has backward incompatible changes
           # affecting integration tests for this repository
           # The file specified must exist in the Databroker container
-          KDB_OPT: "--vss vss_release_3.1.1.json"
+          # KDB_OPT: "--vss vss_release_3.1.1.json"
           SEAT_TAG: "prerelease"
           HVAC_TAG: "prerelease"
         run: |

.github/workflows/scripts/install-ci-tooling.sh

@@ -17,31 +17,34 @@ export ENV DEBIAN_FRONTEND=noninteractive
 
 # Install apt & root utils needed inside devcontainer
 sudo apt-get -qqy update &&
-	sudo apt-get -qqy install --no-install-recommends apt-utils software-properties-common sudo curl
+	sudo apt-get -qqy install --no-install-recommends apt-utils software-properties-common sudo curl file
 
 # Install dev utilities
-sudo apt-get -qqy install git doxygen can-utils python3
+sudo apt-get -qqy install --no-install-recommends git doxygen can-utils python3
 
 # Install build tools
-sudo apt-get -qqy install \
+sudo apt-get -qqy install --no-install-recommends \
 	cmake \
 	make
 
 # compilers
 #   compiler version must be sync with conan build profile
-sudo apt-get -qqy install \
+sudo apt-get -qqy install --no-install-recommends \
 	g++ \
 	g++-aarch64-linux-gnu
 
-sudo apt-get -qqy install \
+sudo apt-get -qqy install --no-install-recommends \
 	lcov \
 	gcovr \
 	clang-format \
 	cppcheck \
 	valgrind
 
 # Install PIP
-[ -z "$(which pip3)" ] && sudo apt-get -qqy install --fix-missing python3-pip
+[ -z "$(which pip3)" ] && sudo apt-get -qqy install --no-install-recommends --fix-missing python3-pip
+
+# Default pip and setuptools are too old...
+pip3 install --upgrade pip setuptools
 
 # conan: dependency management
 # - conan needed > 1.43 for gtest

.github/workflows/seat_service_seatctrl_test.yml

@@ -50,7 +50,9 @@ jobs:
 
       - name: cobertura-report
         if: ${{ github.event.pull_request.head.repo.full_name == github.repository }}
-        uses: 5monkeys/cobertura-action@v13
+        # NOTE: cobertura-action have a fix for node 12 deprecation warning, but it's not released yet.
+        # Using a pinned commit instead of master for security purposes, to be updated if they release new version
+        uses: 5monkeys/cobertura-action@18d911b62098eafd8b9db93d6c74194556cc3515
         with:
           path: ${{github.workspace}}/seat_service/build_seat_controller/x86_64/report_codecov_vservice-seat-ctrl.xml
           repo_token: ${{ secrets.GITHUB_TOKEN }}

.vscode/scripts/run-databroker-cli.sh

@@ -29,11 +29,16 @@ fi
 
 # https://github.com/eclipse/kuksa.val/releases/download/0.3.0/databroker-cli-amd64.tar.gz
 
+DATABROKER_BINARY_ZIP="databroker-cli-$PROCESSOR_ALT.zip"
+DATABROKER_BINARY_NAME="databroker-cli-$PROCESSOR_ALT.tar.gz"
+
+DOWNLOAD_URL="https://github.com/eclipse/kuksa.val/releases/download/$DATABROKER_VERSION/$DATABROKER_BINARY_ZIP"
+#DOWNLOAD_URL="https://github.com/eclipse/kuksa.val/releases/download/$DATABROKER_VERSION/$DATABROKER_BINARY_NAME"
+
 DATABROKER_BINARY_NAME="databroker-cli-$PROCESSOR_ALT.tar.gz"
 DATABROKER_BINARY_PATH="$ROOT_DIRECTORY/.vscode/scripts/assets/databroker/$DATABROKER_VERSION/$PROCESSOR_ALT"
-DOWNLOAD_URL="https://github.com/eclipse/kuksa.val/releases/download/$DATABROKER_VERSION/$DATABROKER_BINARY_NAME"
 DATABROKERCLI_EXECUTABLE="$DATABROKER_BINARY_PATH/databroker-cli/databroker-cli"
 
-download_release "$DATABROKERCLI_EXECUTABLE" "$DOWNLOAD_URL" "$DATABROKER_BINARY_PATH" "$DATABROKER_BINARY_NAME" || exit 1
+download_zip_release "$DATABROKERCLI_EXECUTABLE" "$DOWNLOAD_URL" "$DATABROKER_BINARY_PATH" "$DATABROKER_BINARY_ZIP" "$DATABROKER_BINARY_NAME" || exit 1
 
 "$DATABROKERCLI_EXECUTABLE"

.vscode/scripts/run-databroker.sh

@@ -41,12 +41,17 @@ if [ -z "$DATABROKER_VERSION" ]; then
 fi
 
 # https://github.com/eclipse/kuksa.val/releases/download/0.3.0/databroker-amd64.tar.gz
+
+### change introduced since databroker 0.3.0 release, zip file containing tar.gz
+# https://github.com/eclipse/kuksa.val/releases/download/0.4.0/databroker-amd64.zip
+
+DATABROKER_BINARY_ZIP="databroker-$PROCESSOR_ALT.zip"
 DATABROKER_BINARY_NAME="databroker-$PROCESSOR_ALT.tar.gz"
 DATABROKER_BINARY_PATH="$ROOT_DIRECTORY/.vscode/scripts/assets/databroker/$DATABROKER_VERSION/$PROCESSOR_ALT"
 DATABROKER_EXECUTABLE="$DATABROKER_BINARY_PATH/databroker/databroker"
-DOWNLOAD_URL="https://github.com/eclipse/kuksa.val/releases/download/$DATABROKER_VERSION/$DATABROKER_BINARY_NAME"
+DOWNLOAD_URL="https://github.com/eclipse/kuksa.val/releases/download/$DATABROKER_VERSION/$DATABROKER_BINARY_ZIP"
 
-download_release "$DATABROKER_EXECUTABLE" "$DOWNLOAD_URL" "$DATABROKER_BINARY_PATH" "$DATABROKER_BINARY_NAME" || exit 1
+download_zip_release "$DATABROKER_EXECUTABLE" "$DOWNLOAD_URL" "$DATABROKER_BINARY_PATH" "$DATABROKER_BINARY_ZIP" "$DATABROKER_BINARY_NAME" || exit 1
 
 ### Data Broker environment setup ###
 
@@ -55,13 +60,17 @@ export RUST_LOG="debug,databroker=debug,vehicle_data_broker=debug,h2=info"
 ##export GRPC_TRACE=all,-timer,-timer_check
 ##export GRPC_VERBOSITY=DEBUG
 
-## Uncomment to preregister vehicle model metadata in vss.json
-DATABROKER_METADATA="--metadata $DATABROKER_BINARY_PATH/vss.json"
+### pre-load vss json versions
+if [ ! -f $DATABROKER_BINARY_PATH/vss3.json ] || [ ! -f $DATABROKER_BINARY_PATH/vss4.json ]; then
+	echo "# Downloading $DATABROKER_BINARY_PATH/vss*.json ..."
+	wget -q "https://raw.githubusercontent.com/eclipse/kuksa.val/master/data/vss-core/vss_release_3.0.json" -O "$DATABROKER_BINARY_PATH/vss3.json"
+	wget -q "https://raw.githubusercontent.com/eclipse/kuksa.val/master/data/vss-core/vss_release_4.0.json" -O "$DATABROKER_BINARY_PATH/vss4.json"
+fi
 
-### handle vss json
-if [ -n "$DATABROKER_METADATA" ] && [ ! -f $DATABROKER_BINARY_PATH/vss.json ]; then
-	echo "# Downloading $DATABROKER_BINARY_PATH/vss.json ..."
-	wget -q "https://raw.githubusercontent.com/eclipse/kuksa.val/master/data/vss-core/vss_release_3.0.json" -O "$DATABROKER_BINARY_PATH/vss.json"
+if [ "$USE_VSS3" = "1" ]; then
+	DATABROKER_METADATA="--metadata $DATABROKER_BINARY_PATH/vss3.json"
+else
+	DATABROKER_METADATA="--metadata $DATABROKER_BINARY_PATH/vss4.json"
 fi
 
 echo
@@ -78,13 +87,14 @@ echo
 ## Uncomment for dapr debug logs
 # DAPR_OPT="--enable-api-logging --log-level debug"
 
+DAPR_OPT="--log-level warn"
 dapr run \
-	--app-id $VEHICLEDATABROKER_DAPR_APP_ID \
+	--app-id "$VEHICLEDATABROKER_DAPR_APP_ID" \
 	--app-protocol grpc \
 	--app-port $DATABROKER_PORT \
 	--dapr-grpc-port $DATABROKER_GRPC_PORT \
 	$DAPR_OPT \
-	--components-path $ROOT_DIRECTORY/.dapr/components \
-	--config $ROOT_DIRECTORY/.dapr/config.yaml \
+	--components-path "$ROOT_DIRECTORY/.dapr/components" \
+	--config "$ROOT_DIRECTORY/.dapr/config.yaml" \
 	& # -- \
 $DATABROKER_EXECUTABLE --address 0.0.0.0 $DATABROKER_METADATA

.vscode/scripts/run-feedercan.sh

@@ -33,6 +33,7 @@ source "$ROOT_DIRECTORY/.vscode/scripts/task-common.sh" "$@"
 # NOTE: use dapr sidecar's grpc port, don't connect directly to sidecar of kdb (DATABROKER_GRPC_PORT)
 export DAPR_GRPC_PORT="$FEEDERCAN_GRPC_PORT"
 
+# NOTE: [config] feedercan.version is git tag for standalone tests, feedercan.imageVersion is for container tests
 FEEDERCAN_VERSION=$(jq -r '.feedercan.version // empty' "$CONFIG_JSON")
 
 ### Default feedercan settings (repo, tag are optional in json)
@@ -47,14 +48,36 @@ FEEDERCAN_DIR="$ROOT_DIRECTORY/.vscode/scripts/assets/feedercan/${FEEDERCAN_VERS
 # path to dbcfeeder within the project
 FEEDERCAN_EXEC_PATH="$FEEDERCAN_DIR/dbc2val"
 
+echo "### Starting from: $FEEDERCAN_EXEC_PATH"
+
 if [ ! -f "$FEEDERCAN_EXEC_PATH/dbcfeeder.py" ]; then
 	[ -d "$FEEDERCAN_DIR" ] || mkdir -p "$FEEDERCAN_DIR"
 	# decide if we want to checkout a branch, or download release sources
 	git_checkout "$FEEDERCAN_REPO" "$FEEDERCAN_DIR" "$FEEDERCAN_VERSION"
 fi
 
 cd "$FEEDERCAN_EXEC_PATH" || exit 1
-pip3 install -q -r requirements.txt
+
+PYTHON_BIN="python3.9"
+PIP3_BIN="pip3.9"
+
+if [ -z "$(which python3.9)" ]; then
+	echo "### WARNING! dbc2val requires python3.9" 1>&2
+	echo "You may install 3.9 in pyenv. Check: https://github.com/pyenv/pyenv#installation"
+	echo "Or using apt:"
+	echo "  sudo apt-get install -y python3.9"
+	echo "  curl https://bootstrap.pypa.io/get-pip.py | python3.9"
+	echo "  python3.9 --version"
+	echo "  pip3.9 --version"
+	echo
+	# fallback to default python (e.g. it may be 3.10)
+	PYTHON_BIN="python3"
+	PIP3_BIN="pip3"
+fi
+
+# PIP_OPT="--upgrade --retries 1 --timeout 3"
+### IMPORTANT: dbc2val now requires python 3.9, does not work on python 3.8
+"$PIP3_BIN" install $PIP_OPT -q -r requirements.txt -r requirements-dev.txt
 
 ####################################
 ### feedercan environment setup ####
@@ -65,14 +88,18 @@ CONFIG_DIR="$ROOT_DIRECTORY/integration_test/volumes/dbc2val"
 
 ### Override default files for feedercan to be consistent with tests
 export DBC_FILE="$CONFIG_DIR/it-can.dbc"
-export MAPPING_FILE="$CONFIG_DIR/it-mapping.yml"
+export MAPPING_FILE="$CONFIG_DIR/it-vss_4.0.json"
+# uncomment if you need to test with DogMode datapoints (requires custom vss config for databroker)
+#export MAPPING_FILE="$CONFIG_DIR/it-vss_4.0-dogmode.json"
 export CANDUMP_FILE="$CONFIG_DIR/it-candump0.log"
-export USECASE="databroker"
 
+# export USECASE="databroker"
+export USE_DBC2VAL=1
+export NO_USE_VAL2DBC=1
+
+export LOG_LEVEL="info" #,dbcfeeder=debug"
 ## Uncomment to enable most of the debug modules
-# export LOG_LEVEL="info,databroker=debug,dbcfeeder.broker_client=debug,dbcfeeder=debug,dbcreader=debug"
-## Uncomment for verbose dbcreader decoded can dumps, needs dbcreader=debug
-# export CAN_VERBOSE=1
+#export LOG_LEVEL="debug,dbcfeeder=debug,kuksa_client.grpc=debug,dbcfeederlib.canplayer=debug,dbcfeederlib.canreader=debug,dbcfeederlib.dbc2vssmapper=debug"
 
 echo
 echo "*******************************************"
@@ -86,13 +113,13 @@ echo
 
 ## Uncomment for dapr debug logs
 # DAPR_OPT="--enable-api-logging --log-level debug"
-
+DAPR_OPT="--log-level warn"
 dapr run \
 	--app-id "$FEEDERCAN_DAPR_APP_ID" \
 	--app-protocol grpc \
 	--dapr-grpc-port $FEEDERCAN_GRPC_PORT \
 	$DAPR_OPT \
-	--components-path "$ROOT_DIRECTORY/.dapr/components" \
+	--resources-path "$ROOT_DIRECTORY/.dapr/components" \
 	--config "$ROOT_DIRECTORY/.dapr/config.yaml" &
 #--
-python3 -u ./dbcfeeder.py
+"$PYTHON_BIN" -u ./dbcfeeder.py

.vscode/scripts/run-hvacservice.sh

@@ -54,7 +54,7 @@ echo
 
 ## uncomment for dapr debug logs
 # DAPR_OPT="--enable-api-logging --log-level debug"1
-
+DAPR_OPT="--log-level warn"
 dapr run \
 	--app-id $HVACSERVICE_DAPR_APP_ID \
 	--app-protocol grpc \

.vscode/scripts/run-kuksa-cli.sh

@@ -21,10 +21,16 @@ ROOT_DIRECTORY=$(git rev-parse --show-toplevel)
 # shellcheck source=/dev/null
 source "$ROOT_DIRECTORY/.vscode/scripts/task-common.sh" "$@"
 
+KUKSACLIENT_VERSION=$(jq -r '.kuksa_client.version // empty' "$CONFIG_JSON")
+if [ -z "$KUKSACLIENT_VERSION" ]; then
+	echo "Coudln't find kuksa-client version from $CONFIG_JSON"
+	exit 1
+fi
+
 echo
-echo "Seat Service examples:"
-echo "  getMetaData Vehicle.Cabin.Seat.Row1.Pos1.Position"
-echo "  setTargetValue Vehicle.Cabin.Seat.Row1.Pos1.Position 200"
+echo "Seat Service examples [VSS4]:"
+echo "  getMetaData Vehicle.Cabin.Seat.Row1.DriverSide.Position"
+echo "  setTargetValue Vehicle.Cabin.Seat.Row1.DriverSide.Position 200"
 echo
 
 # set to 1 to use pypy kuksa-val package
@@ -36,5 +42,5 @@ if [ "$STANDALONE" = "1" ]; then
 	# prevent warning dumps
 	GRPC_ENABLE_FORK_SUPPORT=true kuksa-client --ip 127.0.0.1 --port "$DATABROKER_PORT" --protocol grpc --insecure
 else
-	docker run -it --rm -e GRPC_ENABLE_FORK_SUPPORT=true --net=host ghcr.io/eclipse/kuksa.val/kuksa-client:0.3.0 --ip 127.0.0.1 --port "$DATABROKER_PORT" --protocol grpc --insecure
+	docker run -it --rm -e GRPC_ENABLE_FORK_SUPPORT=true --net=host "ghcr.io/eclipse/kuksa.val/kuksa-client:$KUKSACLIENT_VERSION" # get--ip 127.0.0.1 --port "$DATABROKER_PORT" --protocol grpc #--insecure
 fi

.vscode/scripts/run-mockservice.sh

@@ -54,7 +54,7 @@ echo
 
 ## uncomment for dapr debug logs
 # DAPR_OPT="--enable-api-logging --log-level debug"1
-
+DAPR_OPT="--log-level warn"
 dapr run \
 	--app-id $MOCKSERVICE_DAPR_APP_ID \
 	--app-protocol grpc \

.vscode/scripts/run-seatservice.sh

@@ -89,14 +89,14 @@ echo
 
 ## Uncomment for dapr debug logs
 # DAPR_OPT="--enable-api-logging --log-level debug"
-
+DAPR_OPT="--log-level warn"
 dapr run \
 	--app-id "$SEATSERVICE_DAPR_APP_ID" \
 	--app-protocol grpc \
 	--app-port $SEATSERVICE_PORT \
 	--dapr-grpc-port $SEATSERVICE_GRPC_PORT \
 	$DAPR_OPT \
-	--components-path $ROOT_DIRECTORY/.dapr/components \
-	--config $ROOT_DIRECTORY/.dapr/config.yaml \
-	-- \
-	$SEATSERVICE_EXEC_PATH/val_start.sh
+	--components-path "$ROOT_DIRECTORY/.dapr/components" \
+	--config "$ROOT_DIRECTORY/.dapr/config.yaml" \
+	& # -- \
+"$SEATSERVICE_EXEC_PATH/val_start.sh"