From 8d4f51179c3145cbd39693f0f7d3c99b079f9bc3 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Tue, 17 Mar 2026 10:59:55 -0400 Subject: [PATCH 1/4] ci: allow installing dd-trace-py pipeline artifacts --- scripts/build_layers.sh | 40 ++++++++++++++++++++++++++++++++-------- 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/scripts/build_layers.sh b/scripts/build_layers.sh index f1500275..49e3ea05 100755 --- a/scripts/build_layers.sh +++ b/scripts/build_layers.sh @@ -20,6 +20,9 @@ # DD_TRACE_COMMIT Specific dd-trace-py commit SHA to build from GitHub. # DD_TRACE_COMMIT_BRANCH dd-trace-py branch name to build from GitHub. # DD_TRACE_WHEEL Path to a pre-built ddtrace .whl file. +# UPSTREAM_PIPELINE_ID GitLab pipeline ID from dd-trace-py. Downloads the +# matching pre-built manylinux2014 wheel from S3 for +# each python/arch combination. # # Examples: # # Build a single layer for Python 3.12 on arm64 @@ -91,14 +94,6 @@ replace_ddtrace_dep() { perl -i -0777 -pe "s|ddtrace = \[[^\]]*\]|$1|gs" pyproject.toml } -# Replace ddtrace source if necessary -if [ -n "$DD_TRACE_COMMIT" ]; then - replace_ddtrace_dep "ddtrace = { git = \"https://github.com/DataDog/dd-trace-py.git\", rev = \"$DD_TRACE_COMMIT\" }" -elif [ -n "$DD_TRACE_COMMIT_BRANCH" ]; then - replace_ddtrace_dep "ddtrace = { git = \"https://github.com/DataDog/dd-trace-py.git\", branch = \"$DD_TRACE_COMMIT_BRANCH\" }" -elif [ -n "$DD_TRACE_WHEEL" ]; then - replace_ddtrace_dep "ddtrace = { file = \"$DD_TRACE_WHEEL\" }" -fi function make_path_absolute { echo "$(cd "$(dirname "$1")"; pwd)/$(basename "$1")" } @@ -109,6 +104,35 @@ function docker_build_zip { destination=$(make_path_absolute $2) arch=$3 + # Restore pyproject.toml to a clean state for each build iteration + cp pyproject.toml.bak pyproject.toml + + # Remove any previously downloaded wheels + rm -f ddtrace-*.whl + + # Replace ddtrace source if necessary + if [ -n "$DD_TRACE_COMMIT" ]; then + replace_ddtrace_dep "ddtrace = { git = \"https://github.com/DataDog/dd-trace-py.git\", rev = \"$DD_TRACE_COMMIT\" }" + elif [ -n "$DD_TRACE_COMMIT_BRANCH" ]; then + replace_ddtrace_dep "ddtrace = { git = \"https://github.com/DataDog/dd-trace-py.git\", branch = \"$DD_TRACE_COMMIT_BRANCH\" }" + elif [ -n "$DD_TRACE_WHEEL" ]; then + replace_ddtrace_dep "ddtrace = { file = \"$DD_TRACE_WHEEL\" }" + elif [ -n "$UPSTREAM_PIPELINE_ID" ]; then + S3_BASE="https://dd-trace-py-builds.s3.amazonaws.com/${UPSTREAM_PIPELINE_ID}" + if [ "${arch}" = "amd64" ]; then + PLATFORM="manylinux2014_x86_64" + else + PLATFORM="manylinux2014_aarch64" + fi + curl -sSfL "${S3_BASE}/download-manylinux2014.sh" | bash -s -- \ + --dest . \ + --python-version "$1" \ + --platform "${PLATFORM}" + WHEEL_FILE=$(ls ddtrace-*.whl | head -n 1) + echo "Using S3 wheel: ${WHEEL_FILE}" + replace_ddtrace_dep "ddtrace = { file = \"${WHEEL_FILE}\" }" + fi + # Install datadogpy in a docker container to avoid the mess from switching # between different python runtimes. temp_dir=$(mktemp -d) From 8fe12be135c92fadaaf11a30a4950a1ddbf5157e Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Tue, 17 Mar 2026 11:43:40 -0400 Subject: [PATCH 2/4] we don't have python/pip available --- scripts/build_layers.sh | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/scripts/build_layers.sh b/scripts/build_layers.sh index 49e3ea05..d0c4670e 100755 --- a/scripts/build_layers.sh +++ b/scripts/build_layers.sh @@ -21,8 +21,8 @@ # DD_TRACE_COMMIT_BRANCH dd-trace-py branch name to build from GitHub. # DD_TRACE_WHEEL Path to a pre-built ddtrace .whl file. # UPSTREAM_PIPELINE_ID GitLab pipeline ID from dd-trace-py. Downloads the -# matching pre-built manylinux2014 wheel from S3 for -# each python/arch combination. +# matching pre-built wheel from S3 (via +# index-manylinux2014.html) for each python/arch. # # Examples: # # Build a single layer for Python 3.12 on arm64 @@ -124,11 +124,15 @@ function docker_build_zip { else PLATFORM="manylinux2014_aarch64" fi - curl -sSfL "${S3_BASE}/download-manylinux2014.sh" | bash -s -- \ - --dest . \ - --python-version "$1" \ - --platform "${PLATFORM}" - WHEEL_FILE=$(ls ddtrace-*.whl | head -n 1) + PY_TAG="cp$(echo "$1" | tr -d '.')" + WHEEL_FILE=$(curl -sSfL "${S3_BASE}/index-manylinux2014.html" \ + | grep -o "ddtrace-[^\"]*${PY_TAG}[^\"]*${PLATFORM}[^\"]*\.whl" \ + | head -n 1) + if [ -z "${WHEEL_FILE}" ]; then + echo "Error: no wheel found for ${PY_TAG} ${PLATFORM} in ${S3_BASE}/index-manylinux2014.html" >&2 + exit 1 + fi + curl -sSfL "${S3_BASE}/${WHEEL_FILE}" -o "${WHEEL_FILE}" echo "Using S3 wheel: ${WHEEL_FILE}" replace_ddtrace_dep "ddtrace = { file = \"${WHEEL_FILE}\" }" fi From 6c458f90009490d9f3151ab632f22b3c98fe96b7 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Tue, 17 Mar 2026 11:53:59 -0400 Subject: [PATCH 3/4] don't fail if wheels aren't available --- scripts/build_layers.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/build_layers.sh b/scripts/build_layers.sh index d0c4670e..b5e15521 100755 --- a/scripts/build_layers.sh +++ b/scripts/build_layers.sh @@ -129,12 +129,12 @@ function docker_build_zip { | grep -o "ddtrace-[^\"]*${PY_TAG}[^\"]*${PLATFORM}[^\"]*\.whl" \ | head -n 1) if [ -z "${WHEEL_FILE}" ]; then - echo "Error: no wheel found for ${PY_TAG} ${PLATFORM} in ${S3_BASE}/index-manylinux2014.html" >&2 - exit 1 + echo "No S3 wheel found for ${PY_TAG} ${PLATFORM}, using default pyproject.toml version" + else + curl -sSfL "${S3_BASE}/${WHEEL_FILE}" -o "${WHEEL_FILE}" + echo "Using S3 wheel: ${WHEEL_FILE}" + replace_ddtrace_dep "ddtrace = { file = \"${WHEEL_FILE}\" }" fi - curl -sSfL "${S3_BASE}/${WHEEL_FILE}" -o "${WHEEL_FILE}" - echo "Using S3 wheel: ${WHEEL_FILE}" - replace_ddtrace_dep "ddtrace = { file = \"${WHEEL_FILE}\" }" fi # Install datadogpy in a docker container to avoid the mess from switching From 659be543cf6a603cf945c7fea556b8f48e5405c9 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Wed, 18 Mar 2026 11:56:01 -0400 Subject: [PATCH 4/4] don't delete local wheels --- scripts/build_layers.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/scripts/build_layers.sh b/scripts/build_layers.sh index b5e15521..7456a38b 100755 --- a/scripts/build_layers.sh +++ b/scripts/build_layers.sh @@ -107,9 +107,6 @@ function docker_build_zip { # Restore pyproject.toml to a clean state for each build iteration cp pyproject.toml.bak pyproject.toml - # Remove any previously downloaded wheels - rm -f ddtrace-*.whl - # Replace ddtrace source if necessary if [ -n "$DD_TRACE_COMMIT" ]; then replace_ddtrace_dep "ddtrace = { git = \"https://github.com/DataDog/dd-trace-py.git\", rev = \"$DD_TRACE_COMMIT\" }"