chore: rearrange some code to prepare for more span pointers (#3207)

Co-authored-by: Christophe Papazian <114495376+christophe-papazian@users.noreply.github.com>
DataDog · Oct 15, 2024 · 2a6b751 · 2a6b751
1 parent be6bac6
commit 2a6b751
Show file tree

Hide file tree

Showing 21 changed files with 160 additions and 129 deletions.
diff --git a/manifests/cpp.yml b/manifests/cpp.yml
@@ -172,8 +172,10 @@ tests/:
       Test_RemoteConfigurationUpdateSequenceLiveDebugging: missing_feature
       Test_RemoteConfigurationUpdateSequenceLiveDebuggingNoCache: missing_feature
   serverless/:
-    test_span_pointers.py:
-      Test_SpanPointers: missing_feature
+    span_pointers/:
+      aws/:
+        test_s3_span_pointers.py:
+          Test_PutObject: missing_feature
   stats/:
     test_miscs.py:
       Test_Miscs: missing_feature

diff --git a/manifests/dotnet.yml b/manifests/dotnet.yml
@@ -379,8 +379,10 @@ tests/:
       Test_RemoteConfigurationUpdateSequenceLiveDebugging: v2.15.0
       Test_RemoteConfigurationUpdateSequenceLiveDebuggingNoCache: irrelevant (cache is implemented)
   serverless/:
-    test_span_pointers.py:
-      Test_SpanPointers: missing_feature
+    span_pointers/:
+      aws/:
+        test_s3_span_pointers.py:
+          Test_PutObject: missing_feature
   stats/:
     test_miscs.py:
       Test_Miscs: missing_feature

diff --git a/manifests/golang.yml b/manifests/golang.yml
@@ -494,8 +494,10 @@ tests/:
       Test_RemoteConfigurationUpdateSequenceLiveDebugging: missing_feature
       Test_RemoteConfigurationUpdateSequenceLiveDebuggingNoCache: irrelevant (cache is implemented)
   serverless/:
-    test_span_pointers.py:
-      Test_SpanPointers: missing_feature
+    span_pointers/:
+      aws/:
+        test_s3_span_pointers.py:
+          Test_PutObject: missing_feature
   test_config_consistency.py:
     Test_Config_ClientIPHeader_Configured: v1.60.0
     Test_Config_ClientIPHeader_Precedence: v1.69.0
@@ -551,4 +553,4 @@ tests/:
     Test_ProductsDisabled: v1.53
     Test_Telemetry: v1.49.0
     Test_TelemetrySCAEnvVar: missing_feature
-    Test_TelemetryV2: v1.49.1
+    Test_TelemetryV2: v1.49.1
diff --git a/manifests/java.yml b/manifests/java.yml
@@ -1287,8 +1287,10 @@ tests/:
         spring-boot-3-native: missing_feature (GraalVM. Tracing support only)
       Test_RemoteConfigurationUpdateSequenceLiveDebuggingNoCache: irrelevant (cache is implemented)
   serverless/:
-    test_span_pointers.py:
-      Test_SpanPointers: missing_feature
+    span_pointers/:
+      aws/:
+        test_s3_span_pointers.py:
+          Test_PutObject: missing_feature
   stats/:
     test_miscs.py:
       Test_Miscs: missing_feature

diff --git a/manifests/nodejs.yml b/manifests/nodejs.yml
@@ -603,8 +603,10 @@ tests/:
       Test_RemoteConfigurationUpdateSequenceLiveDebugging: *ref_5_16_0 #actual version unknown
       Test_RemoteConfigurationUpdateSequenceLiveDebuggingNoCache: irrelevant (cache is implemented)
   serverless/:
-    test_span_pointers.py:
-      Test_SpanPointers: missing_feature
+    span_pointers/:
+      aws/:
+        test_s3_span_pointers.py:
+          Test_PutObject: missing_feature
   stats/:
     test_miscs.py:
       Test_Miscs: missing_feature

diff --git a/manifests/php.yml b/manifests/php.yml
@@ -327,8 +327,10 @@ tests/:
       Test_RemoteConfigurationUpdateSequenceLiveDebugging: missing_feature
       Test_RemoteConfigurationUpdateSequenceLiveDebuggingNoCache: missing_feature
   serverless/:
-    test_span_pointers.py:
-      Test_SpanPointers: missing_feature
+    span_pointers/:
+      aws/:
+        test_s3_span_pointers.py:
+          Test_PutObject: missing_feature
   stats/:
     test_miscs.py:
       Test_Miscs: missing_feature

diff --git a/manifests/python.yml b/manifests/python.yml
@@ -772,8 +772,15 @@ tests/:
       Test_RemoteConfigurationUpdateSequenceLiveDebugging: v2.8.0.dev
       Test_RemoteConfigurationUpdateSequenceLiveDebuggingNoCache: missing_feature
   serverless/:
-    test_span_pointers.py:
-      Test_SpanPointers: v2.14.0
+    span_pointers/:
+      aws/:
+        test_s3_span_pointers.py:
+          # NOTE: fastapi-based async things don't play well with moto
+          Test_PutObject:
+            '*': missing_feature
+            django-poc: v2.14.0
+            flask-poc: v2.14.0
+            python3.12: v2.14.0
   stats/:
     test_miscs.py:
       Test_Miscs: missing_feature

diff --git a/manifests/ruby.yml b/manifests/ruby.yml
@@ -395,8 +395,10 @@ tests/:
       Test_RemoteConfigurationUpdateSequenceLiveDebugging: missing_feature
       Test_RemoteConfigurationUpdateSequenceLiveDebuggingNoCache: missing_feature
   serverless/:
-    test_span_pointers.py:
-      Test_SpanPointers: missing_feature
+    span_pointers/:
+      aws/:
+        test_s3_span_pointers.py:
+          Test_PutObject: missing_feature
   stats/:
     test_miscs.py:
       Test_Miscs: missing_feature

diff --git a/tests/serverless/__init__.py b/tests/serverless/__init__.py
diff --git a/tests/serverless/span_pointers/__init__.py b/tests/serverless/span_pointers/__init__.py
diff --git a/tests/serverless/span_pointers/aws/__init__.py b/tests/serverless/span_pointers/aws/__init__.py
diff --git a/tests/serverless/span_pointers/aws/test_s3_span_pointers.py b/tests/serverless/span_pointers/aws/test_s3_span_pointers.py
@@ -0,0 +1,48 @@
+import json
+
+from utils import weblog, interfaces, rfc, features
+from utils.tools import logger
+from tests.serverless.span_pointers.utils import (
+    POINTER_DIRECTION_DOWNSTREAM,
+    make_single_span_link_validator,
+    standard_hashing_function,
+)
+
+
+def _validate_s3_object_pointer(r):
+    assert r.status_code == 200
+
+    response_content = json.loads(r.text)
+    bucket = r.request.params["bucket"].encode("ascii")
+    key = r.request.params["key"].encode("utf-8")
+    etag = response_content["object"]["e_tag"].encode("ascii")
+
+    assert b'"' not in etag, "boto3 sometimes includes double-quotes in etags"
+
+    logger.info(f"bucket: {bucket}, key: {key}, etag: {etag}")
+
+    interfaces.library.validate_spans(
+        r,
+        validator=make_single_span_link_validator(
+            pointer_kind="aws.s3.object",
+            pointer_direction=POINTER_DIRECTION_DOWNSTREAM,
+            pointer_hash=standard_hashing_function([bucket, key, etag]),
+        ),
+        full_trace=True,
+    )
+
+
+@rfc("https://github.com/DataDog/dd-span-pointer-rules")
+@features.serverless_span_pointers
+class Test_PutObject:
+    def setup_main(self):
+        self.r = weblog.get("/mock_s3/put_object", params={"bucket": "mybucket", "key": "my-key"})
+
+    def test_main(self):
+        _validate_s3_object_pointer(self.r)
+
+    def setup_non_ascii(self):
+        self.r_non_ascii = weblog.get("/mock_s3/put_object", params={"bucket": "mybucket", "key": "some-key.你好"})
+
+    def test_non_ascii(self):
+        _validate_s3_object_pointer(self.r_non_ascii)
diff --git a/tests/serverless/span_pointers/utils.py b/tests/serverless/span_pointers/utils.py
@@ -0,0 +1,63 @@
+from hashlib import sha256
+from typing import NewType
+
+
+PointerHash = NewType("PointerHash", str)
+
+POINTER_DIRECTION_DOWNSTREAM = "d"
+POINTER_DIRECTION_UPSTREAM = "d"
+
+
+# copyied from https://github.com/DataDog/dd-span-pointer-rules/blob/main/test_py/test_rules.py
+def standard_hashing_function(elements: list[bytes]) -> PointerHash:
+    assert isinstance(elements, list)
+    assert elements
+    assert all(isinstance(element, bytes) for element in elements)
+
+    separator = b"|"
+    bits_per_hex_digit = 4
+    desired_bits = 128
+    hex_digits = desired_bits // bits_per_hex_digit
+
+    hex_digest = sha256(separator.join(elements)).hexdigest()
+    assert len(hex_digest) >= hex_digits
+
+    return PointerHash(hex_digest[:hex_digits])
+
+
+def make_single_span_link_validator(
+    pointer_kind: str, pointer_direction: str, pointer_hash: PointerHash,
+):
+    """
+    Make a validator function for use with interfaces.library.validate_spans.
+    The validator checks that there is one and only one span pointer for the
+    pointer_kind and pointer_direction and that its hash matches the
+    pointer_hash.
+    """
+
+    def validator(span):
+        if "span_links" not in span:
+            return
+
+        found_matching = False
+
+        for span_link in span["span_links"]:
+            attributes = span_link.get("attributes", {})
+            if not attributes:
+                continue
+
+            if attributes.get("ptr.kind") != pointer_kind:
+                continue
+
+            if attributes.get("ptr.dir") != pointer_direction:
+                continue
+
+            assert not found_matching
+
+            assert attributes.get("ptr.hash") == pointer_hash
+            found_matching = True
+            # we loop oonwards in caes there's unexpectedly more than one
+
+        return found_matching
+
+    return validator
diff --git a/tests/serverless/test_span_pointers.py b/tests/serverless/test_span_pointers.py
diff --git a/utils/_features.py b/utils/_features.py
@@ -2410,7 +2410,7 @@ def ssi_service_naming(test_object):
     @staticmethod
     def serverless_span_pointers(test_object):
         """
-        Serverless : span_links is correctly reported
+        Serverless : Span Pointers are correctly incorporated in spans
 
         https://feature-parity.us1.prod.dog/#/?feature=328
         """

diff --git a/utils/build/build_python_base_images.sh b/utils/build/build_python_base_images.sh
@@ -3,16 +3,16 @@
 # build and push python base images
 
 
-docker buildx build --load --progress=plain -f utils/build/docker/python/fastapi.base.Dockerfile -t datadog/system-tests:fastapi.base-v3 .
-docker buildx build --load --progress=plain -f utils/build/docker/python/python3.12.base.Dockerfile -t datadog/system-tests:python3.12.base-v4 .
-docker buildx build --load --progress=plain -f utils/build/docker/python/django-poc.base.Dockerfile -t datadog/system-tests:django-poc.base-v3 .
+docker buildx build --load --progress=plain -f utils/build/docker/python/fastapi.base.Dockerfile -t datadog/system-tests:fastapi.base-v4 .
+docker buildx build --load --progress=plain -f utils/build/docker/python/python3.12.base.Dockerfile -t datadog/system-tests:python3.12.base-v5 .
+docker buildx build --load --progress=plain -f utils/build/docker/python/django-poc.base.Dockerfile -t datadog/system-tests:django-poc.base-v4 .
 docker buildx build --load --progress=plain -f utils/build/docker/python/flask-poc.base.Dockerfile -t datadog/system-tests:flask-poc.base-v7 .
 docker buildx build --load --progress=plain -f utils/build/docker/python/uwsgi-poc.base.Dockerfile -t datadog/system-tests:uwsgi-poc.base-v4 .
 
 if [ "$1" = "--push" ]; then
-      docker push datadog/system-tests:fastapi.base-v3
-      docker push datadog/system-tests:python3.12.base-v4
-      docker push datadog/system-tests:django-poc.base-v3
+      docker push datadog/system-tests:fastapi.base-v4
+      docker push datadog/system-tests:python3.12.base-v5
+      docker push datadog/system-tests:django-poc.base-v4
       docker push datadog/system-tests:flask-poc.base-v7
       docker push datadog/system-tests:uwsgi-poc.base-v4
 fi

diff --git a/utils/build/docker/python/django-poc.Dockerfile b/utils/build/docker/python/django-poc.Dockerfile
@@ -1,4 +1,4 @@
-FROM datadog/system-tests:django-poc.base-v3
+FROM datadog/system-tests:django-poc.base-v4
 
 WORKDIR /app
 

diff --git a/utils/build/docker/python/fastapi.Dockerfile b/utils/build/docker/python/fastapi.Dockerfile
@@ -1,4 +1,4 @@
-FROM datadog/system-tests:fastapi.base-v3
+FROM datadog/system-tests:fastapi.base-v4
 
 WORKDIR /app
 

diff --git a/utils/build/docker/python/fastapi.base.Dockerfile b/utils/build/docker/python/fastapi.base.Dockerfile
@@ -7,7 +7,7 @@ RUN apt-get update && apt-get install -y curl git gcc g++ make cmake
 RUN python --version && curl --version
 
 # install python deps
-RUN pip install fastapi uvicorn cryptography==42.0.8 pycryptodome python-multipart boto3==1.34.141 'moto[s3]'==5.0.14
+RUN pip install fastapi uvicorn cryptography==42.0.8 pycryptodome python-multipart
 
 # Install Rust toolchain
 RUN curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain stable -y