GitLab CI for Kubernetes: a blueprint library for many apps and many clusters

# orders/.gitlab-ci.yml — the whole file
include:
  - project: platform/ci-blueprints
    ref: v2.3.0            # a tag, never main
    file:
      - blueprints/orders/routine.yml
      - blueprints/orders/images.yml
      - blueprints/orders/release.yml
      - blueprints/orders/delivery.yml
      - blueprints/orders/alerts.yml

variables:
  SHIP_WORKER: "true"      # units opt in one at a time

# blueprints/orders/routine.yml — what a branch means, for every consumer
stages: [images, release, delivery, alerts]

workflow:
  rules:
    - if: '$CI_COMMIT_BRANCH == "candidate"'
      variables: { RING: preview }
    - if: '$CI_COMMIT_BRANCH == "main"'
      variables: { RING: live }
    - when: never              # anything else creates no pipeline

# Infrastructure retries only. script_failure is deliberately
# absent: a real defect would run three times and look flaky.
default:
  retry:
    max: 2
    when:
      - runner_system_failure
      - stuck_or_timeout_failure
      - api_failure
      - scheduler_failure

# Shell library, pulled into jobs with `!reference [.lib, shell]`.
.lib:
  shell: |
    # probe: 0 = present · 1 = absent · 2 = unknown (caller must stop)
    probe() {   # $1=namespace $2=kind $3=name
      for n in 1 2 3; do
        out=$(kubectl -n "$1" get "$2" "$3" -o name 2>&1 >/dev/null) \
          && return 0
        case "$out" in *NotFound*) return 1 ;; esac
        sleep $((n * 5))
      done
      echo "unknown: $2/$3 in $1" >&2; return 2
    }
    attempt() {
      for n in 1 2 3; do "$@" && return 0; sleep $((n * 5)); done
      return 1
    }

# blueprints/orders/images.yml — bake one image per unit
.bake:
  stage: images
  image: docker:27
  services: [docker:27-dind]
  rules:
    - if: '$RING == "preview"'          # the live ring never bakes
  variables:
    IMAGE: $CI_REGISTRY_IMAGE/$UNIT
  script:
    - echo "$CI_REGISTRY_PASSWORD" | docker login -u "$CI_REGISTRY_USER" \
        --password-stdin "$CI_REGISTRY"
    - docker build -t "$IMAGE:$CI_COMMIT_SHA" \
        --build-arg RING="$RING" \
        -f "$SRC/Dockerfile" "$SRC"
    - docker push "$IMAGE:$CI_COMMIT_SHA"
    - docker tag "$IMAGE:$CI_COMMIT_SHA" "$IMAGE:$RING"
    - docker push "$IMAGE:$RING"

bake_api:
  extends: .bake
  variables: { UNIT: api, SRC: services/api }

bake_web:
  extends: .bake
  variables: { UNIT: web, SRC: web }

bake_worker:
  extends: .bake
  variables: { UNIT: worker, SRC: services/worker }
  rules:
    - if: '$RING == "preview" && $SHIP_WORKER == "true"'

# blueprints/orders/delivery.yml — hand the rollout set to the fleet
.deliver:
  stage: delivery
  rules:
    - if: '$RING =~ /^(preview|live)$/'
  trigger:
    project: platform/fleet
    strategy: depend          # wait for the child; inherit its result
    forward: { pipeline_variables: true }
  variables:
    APP: orders
    RING: $RING
    ORIGIN_SHA: $CI_COMMIT_SHA
    ROLLOUT_SET: |
      [
        {"unit":"api",    "container":"api",    "image":"$API_IMAGE"},
        {"unit":"web",    "container":"web",    "image":"$WEB_IMAGE"},
        {"unit":"worker", "container":"worker", "image":"$WORKER_IMAGE"}
      ]

deliver_north: { extends: .deliver, variables: { CLUSTER: north } }
deliver_south: { extends: .deliver, variables: { CLUSTER: south } }

# blueprints/orders/release.yml
# On main it re-tags preview as live; on candidate it only resolves digests.
release:
  stage: release
  image: gcr.io/go-containerregistry/crane:debug
  script:
    - |
      for unit in api web worker; do
        img="$CI_REGISTRY_IMAGE/$unit"
        if [ "$RING" = "live" ]; then
          digest=$(crane digest "$img:preview")   # what preview runs now
          crane tag "$img@$digest" live           # same bytes, new name
        else
          digest=$(crane digest "$img:$CI_COMMIT_SHA")
        fi
        key=$(echo "$unit" | tr a-z A-Z)
        echo "${key}_IMAGE=$img@$digest" >> images.env
      done
  artifacts:
    reports: { dotenv: images.env }   # *_IMAGE reach the delivery jobs

# platform/fleet/.gitlab-ci.yml — the only pipeline that touches a cluster
workflow:
  rules:
    - if: '$CI_PIPELINE_SOURCE == "pipeline"'   # an app pipeline fired it
    - if: '$CI_PIPELINE_SOURCE == "web"'        # or someone pressed Run
    - when: never

rollout:
  stage: rollout
  image: bitnami/kubectl:1.31
  resource_group: $CLUSTER/$APP/$RING   # one rollout at a time per target
  script:
    - |
      set -euo pipefail
      # one file-type variable per cluster: KUBECONFIG_NORTH, ...
      eval "export KUBECONFIG=\$KUBECONFIG_$(echo "$CLUSTER" | tr a-z A-Z)"
      NS="$APP-$RING"                       # namespaces follow one convention
      DIR="$CLUSTER/$APP/$RING"

      kubectl -n "$NS" apply -f "$CLUSTER/$APP/common/$RING/"
      kubectl -n "$NS" apply -f "$DIR/"

      echo "$ROLLOUT_SET" \
        | jq -c '.[] | select(.image | test("@sha256"))' \
        | while read -r entry; do
            unit=$(jq -r .unit <<<"$entry")
            container=$(jq -r .container <<<"$entry")
            image=$(jq -r .image <<<"$entry")
            kubectl -n "$NS" set image deployment \
              -l "app=$APP,unit=$unit" "$container=$image"
          done

      for d in $(kubectl -n "$NS" get deploy -l "app=$APP" -o name); do
        kubectl -n "$NS" rollout status "$d" --timeout=10m || {
          kubectl -n "$NS" describe "$d"
          kubectl -n "$NS" logs "$d" --all-containers --tail=100 || true
          exit 1
        }
      done

# blueprints/orders/alerts.yml
alert_failure:
  stage: alerts
  image: alpine:3.20
  rules:
    - when: on_failure
  script:
    - apk add --no-cache curl jq >/dev/null
    - |
      [ -n "${SLACK_WEBHOOK_URL:-}" ] \
        || { echo "no SLACK_WEBHOOK_URL; skipping"; exit 0; }
      title=":red_circle: orders — failed on $CI_COMMIT_REF_NAME"
      msg=$(printf '%s' "$CI_COMMIT_MESSAGE" | head -c 300)
      PAYLOAD=$(jq -n --arg title "$title ($RING)" \
        --arg who "${GITLAB_USER_NAME:-?}" --arg sha "$CI_COMMIT_SHORT_SHA" \
        --arg msg "$msg" --arg url "$CI_PIPELINE_URL" '{
          blocks: [
            { type: "header",
              text: { type: "plain_text", text: $title } },
            { type: "section", fields: [
                { type: "mrkdwn", text: ("*Author*\n" + $who) },
                { type: "mrkdwn", text: ("*Commit*\n`" + $sha + "`") } ] },
            { type: "section",
              text: { type: "mrkdwn", text: $msg } },
            { type: "actions", elements: [
                { type: "button", url: $url,
                  text: { type: "plain_text", text: "Open pipeline" } } ] }
          ] }')
      curl -sS -o /dev/null -w "slack %{http_code}\n" -X POST \
           -H 'Content-type: application/json' \
           -d "$PAYLOAD" "$SLACK_WEBHOOK_URL"

# end of the fleet rollout job — success only; a failure exited above
digest=$(kubectl -n "$NS" get deploy -l "app=$APP,unit=api" \
  -o jsonpath='{.items[0].spec.template.spec.containers[0].image}' \
  | sed 's/.*@sha256://' | head -c 12)
jq -n --arg t ":large_green_circle: orders → $RING on $CLUSTER" \
      --arg d "api @ $digest…" --arg u "$CI_JOB_URL" \
  '{blocks:[{type:"section",text:{type:"mrkdwn",
     text:("*"+$t+"*\n"+$d+"  <"+$u+"|job>")}}]}' \
  | curl -sS -o /dev/null -X POST -H 'Content-type: application/json' \
         -d @- "$SLACK_WEBHOOK_URL" || true