stages:
  - build
  - test
  - gate
  - report

variables:
  BENCHMARKING_IMAGE_REGISTRY: 486234852809.dkr.ecr.us-east-1.amazonaws.com
  MICROBENCHMARKS_CI_IMAGE: $BENCHMARKING_IMAGE_REGISTRY/ci/benchmarking-platform:dd-trace-py
  PACKAGE_IMAGE: registry.ddbuild.io/images/mirror/pypa/manylinux2014_x86_64:2025-04-12-5990e2d
  GITHUB_CLI_IMAGE: registry.ddbuild.io/images/dd-octo-sts-ci-base:2025.06-1
  BENCHMARKING_BRANCH: dd-trace-py

.benchmarks:
  stage: test
  when: on_success
  tags: ["runner:apm-k8s-tweaked-metal"]
  image: $MICROBENCHMARKS_CI_IMAGE
  interruptible: true
  timeout: 30m
  dependencies: [ "baseline:build", "candidate" ]
  script: |
    export REPORTS_DIR="$(pwd)/reports/" && (mkdir "${REPORTS_DIR}" || :)

    if [[ -n "$CI_JOB_TOKEN" ]];
    then
      git config --global url."https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/".insteadOf "https://github.com/DataDog/"
    fi
    git clone --branch "${BENCHMARKING_BRANCH}" https://github.com/DataDog/benchmarking-platform /platform
    export PATH="$PATH:/platform/steps"

    capture-hardware-software-info.sh

    if [[ $SCENARIO =~ ^flask_* || $SCENARIO =~ ^django_* ]];
    then
      BP_SCENARIO=$SCENARIO bp-runner "${CI_PROJECT_DIR:-.}/.gitlab/benchmarks/bp-runner.yml" --debug -t
    else
      run-benchmarks.sh
    fi

    analyze-results.sh

    upload-results-to-s3.sh || :

    # We have to move artifacts to ${CI_PROJECT_DIR} if we want to attach as GitLab artifact
    cp -R /artifacts ${CI_PROJECT_DIR}/
  artifacts:
    name: "reports"
    when: always
    paths:
      - reports/
      - artifacts/
    expire_in: 3 months
  variables:
    UPSTREAM_PROJECT_ID: $CI_PROJECT_ID # The ID of the current project. This ID is unique across all projects on the GitLab instance.
    UPSTREAM_PROJECT_NAME: $CI_PROJECT_NAME # "dd-trace-py"
    UPSTREAM_BRANCH: $CI_COMMIT_REF_NAME # The branch or tag name for which project is built.
    UPSTREAM_COMMIT_SHA: $CI_COMMIT_SHA # The commit revision the project is built for.
    CARGO_NET_GIT_FETCH_WITH_CLI: "true" # use system git binary to pull git dependencies
    CMAKE_BUILD_PARALLEL_LEVEL: 12
    CARGO_BUILD_JOBS: 12

baseline:detect:
  image: $GITHUB_CLI_IMAGE
  tags: [ "arch:amd64" ]
  stage: build
  id_tokens:
    DDOCTOSTS_ID_TOKEN:
      aud: dd-octo-sts
  variables:
    UPSTREAM_BRANCH: $CI_COMMIT_REF_NAME
  script: |
    GITHUB_REPO_URL="https://github.com/DataDog/dd-trace-py.git"
    git config --global --add safe.directory "${GITHUB_REPO_URL}"
    git config --global --add safe.directory ${CI_PROJECT_DIR}
    git remote set-url origin "${GITHUB_REPO_URL}"

    if [ -z ${GH_TOKEN} ]
    then
      # Use dd-octo-sts to get GitHub token
      dd-octo-sts token --scope DataDog/dd-trace-py --policy gitlab.github-access.read > token
      gh auth login --with-token < token
      rm token
    fi

    # Determine baseline to test against and save env variables into `baseline.env`
    .gitlab/benchmarks/steps/detect-baseline.sh
  artifacts:
    reports:
      dotenv: baseline.env
    paths:
      - "baseline.env"

baseline:build:
  image: $PACKAGE_IMAGE
  tags: [ "arch:amd64" ]
  needs: [ "baseline:detect" ]
  stage: build
  variables:
    CMAKE_BUILD_PARALLEL_LEVEL: 12
    CARGO_BUILD_JOBS: 12
  script: |
    CACHED_WHL=$(ls *.whl | head -n 1) 2>/dev/null || echo ""
    if [ ! -f "${CACHED_WHL}" ];
    then
      .gitlab/benchmarks/steps/build-baseline.sh
    else
      echo "Using wheel from cache for ${BASELINE_BRANCH}:${BASELINE_COMMIT_SHA}:${CACHED_WHL}"
    fi

    echo "BASELINE_WHL=$(ls *.whl | head -n 1)" | tee -a baseline.env
  cache:
    - key: v0-microbenchmarks-baseline-build-${BASELINE_COMMIT_SHA}
      paths:
        - "*.whl"
  artifacts:
    reports:
      dotenv: baseline.env
    paths:
      - "*.whl"

candidate:
  image: $PACKAGE_IMAGE
  stage: build
  tags: [ "arch:amd64" ]
  needs:
    - pipeline: $PARENT_PIPELINE_ID
      job: download_ddtrace_artifacts
  script: |
    cp pywheels/*-cp39-cp39-manylinux*_x86_64*.whl ./
    echo "CANDIDATE_WHL=$(ls *.whl | head -n 1)" | tee candidate.env
    echo "CANDIDATE_BRANCH=${CI_COMMIT_REF_NAME}" | tee -a candidate.env
    echo "CANDIDATE_COMMIT_SHA=${CI_COMMIT_SHA}" | tee -a candidate.env
    echo "CANDIDATE_COMMIT_DATE=$(git show -s --format=%ct $CI_COMMIT_SHA)" | tee -a candidate.env
  artifacts:
    reports:
      dotenv: candidate.env
    paths:
      - "*.whl"

microbenchmarks:
  extends: .benchmarks
  parallel:
    matrix:
      - SCENARIO:
        - "span"
        - "tracer"
        - "sampling_rule_matches"
        - "set_http_meta"
        - "django_simple"
        - "flask_simple"
        - "flask_sqli"
        - "core_api"
        - "otel_span"
        - "otel_sdk_span"
        - "appsec_iast_aspects"
        - "appsec_iast_aspects_ospath"
        - "appsec_iast_aspects_re_module"
        - "appsec_iast_aspects_split"
        # Flaky timeouts on starting up
        # - "appsec_iast_django_startup"
        # TOOD: Re-enable when this issue is resolved:
        #   AttributeError: 'OverheadControl' object has no attribute 'release_request'
        # - "appsec_iast_propagation"
        - "errortracking_django_simple"
        # They take a long time to run and frequently time out
        # TODO: Make benchmarks faster, or run less frequently, or as macrobenchmarks
        # - "appsec_iast_django_startup"
        - "errortracking_flask_sqli"
        # Flaky. Timeout errors
        # - "encoder"
        - "http_propagation_extract"
        - "http_propagation_inject"
        - "rate_limiter"
        - "packages_package_for_root_module_mapping"
        - "packages_update_imported_dependencies"
        - "recursive_computation"
        - "telemetry_add_metric"
        # They take a long time to run, and now need the agent running
        # TODO: Make benchmarks faster, or run less frequently, or as macrobenchmarks
        # - "startup"

benchmarks-pr-comment:
  image: $MICROBENCHMARKS_CI_IMAGE
  tags: ["arch:amd64"]
  stage: report
  when: always
  allow_failure: true
  script: |
    export REPORTS_DIR="$(pwd)/reports/" && (mkdir "${REPORTS_DIR}" || :)
    if [[ -n "$CI_JOB_TOKEN" ]];
    then
      git config --global url."https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/".insteadOf "https://github.com/DataDog/"
    fi
    git clone --branch "${BENCHMARKING_BRANCH}" https://github.com/DataDog/benchmarking-platform /platform
    export PATH="$PATH:/platform/steps"

    (for i in {1..2}; do upload-results-to-benchmarking-api.sh && break; done) || :
    if [ "$CI_COMMIT_REF_NAME" != "main" ];
    then
      post-pr-comment.sh || :
    fi
  variables:
    UPSTREAM_PROJECT_ID: $CI_PROJECT_ID # The ID of the current project. This ID is unique across all projects on the GitLab instance.
    UPSTREAM_PROJECT_NAME: $CI_PROJECT_NAME # "dd-trace-py"
    UPSTREAM_BRANCH: $CI_COMMIT_REF_NAME # The branch or tag name for which project is built.
    UPSTREAM_COMMIT_SHA: $CI_COMMIT_SHA # The commit revision the project is built for.

check-slo-breaches:
  stage: gate
  when: always
  tags: ["arch:amd64"]
  image: registry.ddbuild.io/images/benchmarking-platform-tools-ubuntu:latest
  artifacts:
    name: "artifacts"
    when: always
    paths:
      - artifacts/
      - reports/
    expire_in: 3 months
  script:
    - export ARTIFACTS_DIR="$(pwd)/reports/"
    - bp-runner .gitlab/benchmarks/bp-runner.microbenchmarks.fail-on-breach.yml
