stages:
  - build
  - test
  - gate
  - notify

include:
  - project: 'DataDog/benchmarking-platform-tools'
    file: 'images/templates/gitlab/notify-slo-breaches.template.yml'
  - project: 'DataDog/benchmarking-platform-tools'
    file: 'images/templates/gitlab/check-slo-breaches.template.yml'

variables:
  MACROBENCHMARKS_CI_IMAGE: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/benchmarking-platform:dd-trace-py-macrobenchmarks

candidate:
  image: $PACKAGE_IMAGE
  stage: build
  tags: [ "arch:amd64" ]
  needs:
    - pipeline: $PARENT_PIPELINE_ID
      job: download_ddtrace_artifacts
  script: |
    cp pywheels/*-cp39-cp39-manylinux*_x86_64*.whl ./
    echo "CANDIDATE_WHL=$(ls *.whl | head -n 1)" | tee candidate.env
  artifacts:
    reports:
      dotenv: candidate.env
    paths:
      - "*.whl"

.macrobenchmarks:
  stage: test
  needs: [ "candidate" ]
  tags: ["runner:apm-k8s-same-cpu"]
  timeout: 1h
  retry:
    max: 2
    when:
      - unknown_failure
      - data_integrity_failure
      - runner_system_failure
      - scheduler_failure
      - api_failure
  when: on_success
  image: $MACROBENCHMARKS_CI_IMAGE
  script: |
    git clone --branch python/macrobenchmarks https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/benchmarking-platform platform && cd platform
    if [ "$BP_PYTHON_SCENARIO_DIR" == "flask-realworld" ]; then
      bp-runner bp-runner.flask-realworld.yml --debug
    else
      bp-runner bp-runner.simple.yml --debug
    fi
  artifacts:
    name: "artifacts"
    when: always
    paths:
      - platform/artifacts/
    expire_in: 3 months
  variables:
    DDTRACE_INSTALL_VERSION: "${CI_PROJECT_DIR}/${CANDIDATE_WHL}"

    # Benchmark's env variables. Modify to tweak benchmark parameters.
    DD_TRACE_DEBUG: "false"
    DD_RUNTIME_METRICS_ENABLED: "false"
    DD_REMOTE_CONFIGURATION_ENABLED: "false"
    DD_INSTRUMENTATION_TELEMETRY_ENABLED: "false"
    DD_CRASHTRACKING_ENABLED: "false"
    DD_APPSEC_ENABLED: "false"
    _DD_TRACE_WRITER_NATIVE: "false"

    K6_OPTIONS_WARMUP_RATE: 40
    K6_OPTIONS_WARMUP_DURATION: 1m
    K6_OPTIONS_WARMUP_GRACEFUL_STOP: 0s
    K6_OPTIONS_WARMUP_PRE_ALLOCATED_VUS: 4
    K6_OPTIONS_WARMUP_MAX_VUS: 4

    K6_OPTIONS_NORMAL_OPERATION_RATE: 40
    K6_OPTIONS_NORMAL_OPERATION_DURATION: 5m
    K6_OPTIONS_NORMAL_OPERATION_GRACEFUL_STOP: 1m
    K6_OPTIONS_NORMAL_OPERATION_PRE_ALLOCATED_VUS: 4
    K6_OPTIONS_NORMAL_OPERATION_MAX_VUS: 4

    K6_OPTIONS_HIGH_LOAD_RATE: 500
    K6_OPTIONS_HIGH_LOAD_DURATION: 1m
    K6_OPTIONS_HIGH_LOAD_GRACEFUL_STOP: 30s
    K6_OPTIONS_HIGH_LOAD_PRE_ALLOCATED_VUS: 4
    K6_OPTIONS_HIGH_LOAD_MAX_VUS: 4

  # Workaround: Currently we're not running the benchmarks on every PR, but GitHub still shows them as pending.
  # By marking the benchmarks as allow_failure, this should go away. (This workaround should be removed once the
  # benchmarks get changed to run on every PR)
  allow_failure: true

baseline:
  extends: .macrobenchmarks
  variables:
    DD_BENCHMARKS_CONFIGURATION: baseline
    BP_PYTHON_SCENARIO_DIR: flask-realworld
    BP_ENDPOINT: http://0.0.0.0:8000/api/articles
    DD_SERVICE: "ddtrace-mb-baseline"


tracing-runtime-metrics-enabled:
  extends: .macrobenchmarks
  variables:
    DD_BENCHMARKS_CONFIGURATION: only-tracing
    BP_PYTHON_SCENARIO_DIR: flask-realworld
    BP_ENDPOINT: http://0.0.0.0:8000/api/articles
    DD_TRACE_ENABLED: "true"
    DD_RUNTIME_METRICS_ENABLED: "false"
    DD_SERVICE: "ddtrace-mb-tracing-runtime-metrics-enabled"

tracing-rc-disabled-telemetry-disabled:
  extends: .macrobenchmarks
  variables:
    DD_BENCHMARKS_CONFIGURATION: only-tracing
    BP_PYTHON_SCENARIO_DIR: flask-realworld
    BP_ENDPOINT: http://0.0.0.0:8000/api/articles
    DD_TRACE_ENABLED: "true"
    DD_PROFILING_ENABLED: "false"
    DD_REMOTE_CONFIGURATION_ENABLED: "false"
    DD_INSTRUMENTATION_TELEMETRY_ENABLED: "false"
    DD_SERVICE: "ddtrace-mb-tracing-rc-disabled-telemetry-disabled"

tracing-rc-enabled-telemetry-disabled:
  extends: .macrobenchmarks
  variables:
    DD_BENCHMARKS_CONFIGURATION: only-tracing
    BP_PYTHON_SCENARIO_DIR: flask-realworld
    BP_ENDPOINT: http://0.0.0.0:8000/api/articles
    DD_TRACE_ENABLED: "true"
    DD_PROFILING_ENABLED: "false"
    DD_REMOTE_CONFIGURATION_ENABLED: "true"
    DD_INSTRUMENTATION_TELEMETRY_ENABLED: "false"
    DD_SERVICE: "ddtrace-mb-tracing-rc-enabled-telemetry-disabled"

tracing-rc-disabled-telemetry-enabled:
  extends: .macrobenchmarks
  variables:
    DD_BENCHMARKS_CONFIGURATION: only-tracing
    BP_PYTHON_SCENARIO_DIR: flask-realworld
    BP_ENDPOINT: http://0.0.0.0:8000/api/articles
    DD_TRACE_ENABLED: "true"
    DD_PROFILING_ENABLED: "false"
    DD_REMOTE_CONFIGURATION_ENABLED: "false"
    DD_INSTRUMENTATION_TELEMETRY_ENABLED: "true"
    DD_SERVICE: "ddtrace-mb-tracing-rc-disabled-telemetry-enabled"

tracing-rc-enabled-telemetry-enabled:
  extends: .macrobenchmarks
  variables:
    DD_BENCHMARKS_CONFIGURATION: only-tracing
    BP_PYTHON_SCENARIO_DIR: flask-realworld
    BP_ENDPOINT: http://0.0.0.0:8000/api/articles
    DD_TRACE_ENABLED: "true"
    DD_PROFILING_ENABLED: "false"
    DD_REMOTE_CONFIGURATION_ENABLED: "true"
    DD_INSTRUMENTATION_TELEMETRY_ENABLED: "true"
    DD_SERVICE: "ddtrace-mb-tracing-rc-enabled-telemetry-enabled"

tracing-native-writer-enabled:
  extends: .macrobenchmarks
  variables:
    DD_BENCHMARKS_CONFIGURATION: only-tracing
    BP_PYTHON_SCENARIO_DIR: flask-realworld
    BP_ENDPOINT: http://0.0.0.0:8000/api/articles
    DD_TRACE_ENABLED: "true"
    DD_PROFILING_ENABLED: "false"
    DD_INSTRUMENTATION_TELEMETRY_ENABLED: "true"
    _DD_TRACE_WRITER_NATIVE: "true"
    DD_SERVICE: "ddtrace-mb-tracing-native-writer-enabled"

tracing-native-writer-enabled-stats-enabled:
  extends: .macrobenchmarks
  variables:
    DD_BENCHMARKS_CONFIGURATION: only-tracing
    BP_PYTHON_SCENARIO_DIR: flask-realworld
    BP_ENDPOINT: http://0.0.0.0:8000/api/articles
    DD_TRACE_ENABLED: "true"
    DD_PROFILING_ENABLED: "false"
    DD_INSTRUMENTATION_TELEMETRY_ENABLED: "true"
    DD_TRACE_STATS_COMPUTATION_ENABLED: "true"
    _DD_TRACE_WRITER_NATIVE: "true"
    DD_SERVICE: "ddtrace-mb-tracing-native-writer-enabled-stats-enabled"

tracing-profiling-enabled:
  extends: .macrobenchmarks
  variables:
    DD_BENCHMARKS_CONFIGURATION: only-tracing
    BP_PYTHON_SCENARIO_DIR: flask-realworld
    BP_ENDPOINT: http://0.0.0.0:8000/api/articles
    DD_TRACE_ENABLED: "true"
    DD_PROFILING_ENABLED: "true"
    DD_SERVICE: "ddtrace-mb-tracing-profiling-enabled"

appsec-enabled-iast-disabled-ep-disabled:
  extends: .macrobenchmarks
  variables:
    DD_BENCHMARKS_CONFIGURATION: only-tracing
    BP_PYTHON_SCENARIO_DIR: flask-realworld
    BP_ENDPOINT: http://0.0.0.0:8000/api/articles
    DD_TRACE_ENABLED: "true"
    DD_PROFILING_ENABLED: "false"
    DD_APPSEC_ENABLED: "true"
    DD_IAST_ENABLED: "false"
    DD_APPSEC_RASP_ENABLED: "false"
    DD_SERVICE: "ddtrace-mb-appsec-enabled-iast-disabled-ep-disabled"

appsec-enabled-iast-disabled-ep-disabled-profiling-enabled:
  extends: .macrobenchmarks
  variables:
    DD_BENCHMARKS_CONFIGURATION: only-tracing
    BP_PYTHON_SCENARIO_DIR: flask-realworld
    BP_ENDPOINT: http://0.0.0.0:8000/api/articles
    DD_TRACE_ENABLED: "true"
    DD_PROFILING_ENABLED: "true"
    DD_APPSEC_ENABLED: "true"
    DD_IAST_ENABLED: "false"
    DD_APPSEC_RASP_ENABLED: "false"
    DD_SERVICE: "ddtrace-mb-appsec-enabled-iast-disabled-ep-disabled-profiling-enabled"

appsec-disabled-iast-enabled-ep-disabled:
  extends: .macrobenchmarks
  variables:
    DD_BENCHMARKS_CONFIGURATION: only-tracing
    BP_PYTHON_SCENARIO_DIR: flask-realworld
    BP_ENDPOINT: http://0.0.0.0:8000/api/articles
    DD_TRACE_ENABLED: "true"
    DD_PROFILING_ENABLED: "false"
    DD_APPSEC_ENABLED: "false"
    DD_IAST_ENABLED: "true"
    DD_APPSEC_RASP_ENABLED: "false"
    DD_SERVICE: "ddtrace-mb-appsec-disabled-iast-enabled-ep-disabled"

appsec-disabled-iast-enabled-ep-disabled-profiling-enabled:
  extends: .macrobenchmarks
  variables:
    DD_BENCHMARKS_CONFIGURATION: only-tracing
    BP_PYTHON_SCENARIO_DIR: flask-realworld
    BP_ENDPOINT: http://0.0.0.0:8000/api/articles
    DD_TRACE_ENABLED: "true"
    DD_PROFILING_ENABLED: "true"
    DD_APPSEC_ENABLED: "false"
    DD_IAST_ENABLED: "true"
    DD_APPSEC_RASP_ENABLED: "false"
    DD_SERVICE: "ddtrace-mb-appsec-disabled-iast-enabled-ep-disabled-profiling-enabled"

appsec-enabled-iast-enabled-ep-disabled:
  extends: .macrobenchmarks
  variables:
    DD_BENCHMARKS_CONFIGURATION: only-tracing
    BP_PYTHON_SCENARIO_DIR: flask-realworld
    BP_ENDPOINT: http://0.0.0.0:8000/api/articles
    DD_TRACE_ENABLED: "true"
    DD_PROFILING_ENABLED: "false"
    DD_APPSEC_ENABLED: "true"
    DD_IAST_ENABLED: "true"
    DD_APPSEC_RASP_ENABLED: "false"
    DD_SERVICE: "ddtrace-mb-appsec-enabled-iast-enabled-ep-disabled"

appsec-enabled-iast-disabled-ep-enabled:
  extends: .macrobenchmarks
  variables:
    DD_BENCHMARKS_CONFIGURATION: only-tracing
    BP_PYTHON_SCENARIO_DIR: flask-realworld
    BP_ENDPOINT: http://0.0.0.0:8000/api/articles
    DD_TRACE_ENABLED: "true"
    DD_PROFILING_ENABLED: "false"
    DD_APPSEC_ENABLED: "true"
    DD_IAST_ENABLED: "false"
    DD_APPSEC_RASP_ENABLED: "true"
    DD_SERVICE: "ddtrace-mb-appsec-enabled-iast-disabled-ep-enabled"

vuln-baseline:
  extends: .macrobenchmarks
  variables:
    DD_BENCHMARKS_CONFIGURATION: baseline
    BP_PYTHON_SCENARIO_DIR: flask-realworld
    BP_ENDPOINT: http://0.0.0.0:8000/iast/articles?string1=Hi&password=root1234
    IAST_WORST: "true"
    DD_SERVICE: "ddtrace-mb-vuln-baseline"

vuln-appsec-disabled-iast-disabled-ep-disabled:
  extends: .macrobenchmarks
  variables:
    DD_BENCHMARKS_CONFIGURATION: only-tracing
    BP_PYTHON_SCENARIO_DIR: flask-realworld
    BP_ENDPOINT: http://0.0.0.0:8000/iast/articles?string1=Hi&password=root1234
    DD_TRACE_ENABLED: "true"
    DD_PROFILING_ENABLED: "false"
    DD_APPSEC_ENABLED: "false"
    DD_IAST_ENABLED: "disabled"
    DD_APPSEC_RASP_ENABLED: "false"
    IAST_WORST: "true"
    DD_SERVICE: "ddtrace-mb-vuln-appsec-disabled-iast-disabled-ep-disabled"

vuln-appsec-disabled-iast-enabled-ep-disabled:
  extends: .macrobenchmarks
  variables:
    DD_BENCHMARKS_CONFIGURATION: only-tracing
    BP_PYTHON_SCENARIO_DIR: flask-realworld
    BP_ENDPOINT: http://0.0.0.0:8000/iast/articles?string1=Hi&password=root1234
    DD_TRACE_ENABLED: "true"
    DD_PROFILING_ENABLED: "false"
    DD_APPSEC_ENABLED: "false"
    DD_IAST_ENABLED: "true"
    DD_APPSEC_RASP_ENABLED: "false"
    IAST_WORST: "true"
    DD_SERVICE: "ddtrace-mb-vuln-appsec-disabled-iast-enabled-ep-disabled"

vuln-appsec-disabled-iast-enabled-ep-disabled-profiling-enabled:
  extends: .macrobenchmarks
  variables:
    DD_BENCHMARKS_CONFIGURATION: only-tracing
    BP_PYTHON_SCENARIO_DIR: flask-realworld
    BP_ENDPOINT: http://0.0.0.0:8000/iast/articles?string1=Hi&password=root1234
    DD_TRACE_ENABLED: "true"
    DD_PROFILING_ENABLED: "true"
    DD_APPSEC_ENABLED: "false"
    DD_IAST_ENABLED: "true"
    DD_APPSEC_RASP_ENABLED: "false"
    IAST_WORST: "true"
    DD_SERVICE: "ddtrace-mb-vuln-appsec-disabled-iast-enabled-ep-disabled-profiling-enabled"

vuln-appsec-enabled-iast-enabled-ep-disabled:
  extends: .macrobenchmarks
  variables:
    DD_BENCHMARKS_CONFIGURATION: only-tracing
    BP_ENDPOINT: http://0.0.0.0:8000/iast/articles?string1=Hi&password=root1234
    BP_PYTHON_SCENARIO_DIR: flask-realworld
    DD_APPSEC_ENABLED: "true"
    DD_IAST_ENABLED: "true"
    DD_APPSEC_RASP_ENABLED: "false"
    IAST_WORST: "true"
    DD_SERVICE: "ddtrace-mb-vuln-appsec-enabled-iast-enabled-ep-disabled"

attack-appsec-disabled-iast-disabled-ep-disabled:
  extends: .macrobenchmarks
  variables:
    DD_BENCHMARKS_CONFIGURATION: only-tracing
    BP_PYTHON_SCENARIO_DIR: flask-realworld
    BP_ENDPOINT: http://0.0.0.0:8000/api/articles?q=select%20pg_sleep
    DD_TRACE_ENABLED: "true"
    DD_PROFILING_ENABLED: "false"
    DD_APPSEC_ENABLED: "false"
    DD_IAST_ENABLED: "false"
    DD_APPSEC_RASP_ENABLED: "false"
    IAST_WORST: "true"
    DD_SERVICE: "ddtrace-mb-attack-appsec-disabled-iast-disabled-ep-disabled"

attack-appsec-enabled-iast-disabled-ep-enabled:
  extends: .macrobenchmarks
  variables:
    DD_BENCHMARKS_CONFIGURATION: only-tracing
    BP_PYTHON_SCENARIO_DIR: flask-realworld
    BP_ENDPOINT: http://0.0.0.0:8000/api/articles?q=select%20pg_sleep
    DD_TRACE_ENABLED: "true"
    DD_PROFILING_ENABLED: "false"
    DD_APPSEC_ENABLED: "true"
    DD_IAST_ENABLED: "false"
    DD_APPSEC_RASP_ENABLED: "true"
    IAST_WORST: "true"
    DD_SERVICE: "ddtrace-mb-attack-appsec-enabled-iast-disabled-ep-enabled"

attack-appsec-enabled-iast-disabled-ep-enabled-profiling-enabled:
  extends: .macrobenchmarks
  variables:
    DD_BENCHMARKS_CONFIGURATION: only-tracing
    BP_PYTHON_SCENARIO_DIR: flask-realworld
    BP_ENDPOINT: http://0.0.0.0:8000/api/articles?q=select%20pg_sleep
    DD_TRACE_ENABLED: "true"
    DD_PROFILING_ENABLED: "true"
    DD_APPSEC_ENABLED: "true"
    DD_IAST_ENABLED: "false"
    DD_APPSEC_RASP_ENABLED: "true"
    IAST_WORST: "true"
    DD_SERVICE: "ddtrace-mb-attack-appsec-enabled-iast-disabled-ep-enabled-profiling-enabled"

attack-appsec-enabled-iast-enabled-ep-enabled:
  extends: .macrobenchmarks
  variables:
    DD_BENCHMARKS_CONFIGURATION: only-tracing
    BP_PYTHON_SCENARIO_DIR: flask-realworld
    BP_ENDPOINT: http://0.0.0.0:8000/api/articles?q=select%20pg_sleep
    DD_TRACE_ENABLED: "true"
    DD_PROFILING_ENABLED: "false"
    DD_APPSEC_ENABLED: "true"
    DD_IAST_ENABLED: "true"
    DD_APPSEC_RASP_ENABLED: "true"
    IAST_WORST: "true"
    DD_SERVICE: "ddtrace-mb-attack-appsec-enabled-iast-enabled-ep-enabled"

check-slo-breaches:
  extends: .check-slo-breaches
  stage: gate
  when: always
  artifacts:
    name: "artifacts"
    when: always
    paths:
      - platform/artifacts/
    expire_in: 3 months
  variables:
    DDOCTOSTS_POLICY: "gitlab.github-access.read"
    ARTIFACTS_DIR: "platform/artifacts"
    SLO_FILE: ".gitlab/benchmarks/bp-runner.macrobenchmarks.fail-on-breach.yml"

notify-slo-breaches:
  extends: .notify-slo-breaches
  stage: notify
  needs: ["check-slo-breaches"]
  when: always
  variables:
    CHANNEL: "apm-python-release"
