Scaling pipelines: reuse, your own runners, speed and cost

# .github/workflows/node-ci.yml
name: Node CI (reusable)

on:
  workflow_call:
    inputs:
      node-version:
        type: string
        default: '22'
      run-integration:
        type: boolean
        default: true
    outputs:
      coverage-ok:
        description: Whether the coverage threshold passed
        value: ${{ jobs.unit.outputs.coverage-ok }}

permissions:
  contents: read

jobs:
  lint:
    runs-on: ubuntu-latest
    timeout-minutes: 5
    steps:
      - uses: actions/checkout@v7
      - uses: ./.github/actions/setup-shop
        with:
          node-version: ${{ inputs.node-version }}
      - run: npm run lint

  unit:
    runs-on: ubuntu-latest
    timeout-minutes: 10
    outputs:
      coverage-ok: ${{ steps.cov.outcome == 'success' }}
    steps:
      - uses: actions/checkout@v7
      - uses: ./.github/actions/setup-shop
        with:
          node-version: ${{ inputs.node-version }}
      - id: cov
        run: npm run test:coverage

  integration:
    if: inputs.run-integration
    needs: [lint, unit]
    runs-on: ubuntu-latest
    timeout-minutes: 15
    steps:
      - uses: actions/checkout@v7
      - uses: ./.github/actions/setup-shop
        with:
          node-version: ${{ inputs.node-version }}
      - run: npm run test:integration

# .github/workflows/ci.yml
jobs:
  node:
    uses: ./.github/workflows/node-ci.yml             # same repository
    with:
      node-version: '22'

  # from a shared repository, pinned to a tag or SHA, like any action:
  # node:
  #   uses: your-org/ci-templates/.github/workflows/node-ci.yml@v1
  #   with: { run-integration: false }
  #   secrets: inherit

# .github/actions/setup-shop/action.yml
name: Set up shop-api
description: Install Node and the npm dependencies, with caching.
inputs:
  node-version:
    description: Node.js version to install
    default: '22'
runs:
  using: composite
  steps:
    - uses: actions/setup-node@v7
      with:
        node-version: ${{ inputs.node-version }}
        cache: npm
    - run: npm ci
      shell: bash

steps:
  - uses: actions/checkout@v7                 # local actions need the code checked out first
  - uses: ./.github/actions/setup-shop
    with:
      node-version: '22'

runs-on: [self-hosted, linux, x64, gpu]    # labels you gave your runner

services/api/   services/web/   libs/shared/   docs/

#!/usr/bin/env bash
# affected.sh: print the services a change touches. A change in libs/shared touches every service.
base=${1:-main}
changed=$(git diff --name-only "$base"...HEAD)
if grep -q '^libs/shared/' <<<"$changed"; then ls services; exit; fi
grep -o '^services/[^/]*' <<<"$changed" | cut -d/ -f2 | sort -u

$ git diff --name-only main...HEAD          # branch fix-api
services/api/index.js
$ ./affected.sh
api
$ git diff --name-only main...HEAD          # branch shared-change
libs/shared/index.js
$ ./affected.sh
api
web
$ ./affected.sh                             # branch docs-only: nothing to build
$

jobs:
  changes:
    runs-on: ubuntu-latest
    outputs:
      services: ${{ steps.affected.outputs.services }}
    steps:
      - uses: actions/checkout@v7
        with:
          fetch-depth: 0 # the full history, so the diff has a merge base
      - id: affected
        env:
          BASE: origin/${{ github.base_ref }}
        run: echo "services=$(./affected.sh "$BASE" | python3 -c 'import json,sys; print(json.dumps([l.strip() for l in sys.stdin if l.strip()]))')" >> "$GITHUB_OUTPUT"
  test:
    needs: changes
    if: needs.changes.outputs.services != '[]'
    runs-on: ubuntu-latest
    strategy:
      matrix:
        service: ${{ fromJSON(needs.changes.outputs.services) }}
    steps:
      - uses: actions/checkout@v7
      - run: echo "testing services/${{ matrix.service }}"

$ ./affected.sh | python3 -c 'import json,sys; print(json.dumps([l.strip() for l in sys.stdin if l.strip()]))'
["api", "web"]
$ actionlint && echo clean
clean

# dora.py: deploy-* tags are deploys; a "hotfix:" commit marks the deploy before it as failed.
import statistics, subprocess
from datetime import datetime

def git(*args):
    return subprocess.run(["git", *args], capture_output=True, text=True, check=True).stdout.split("\n")

tags = [l.split() for l in git("for-each-ref", "--sort=creatordate", "--format=%(refname:short) %(creatordate:iso-strict)", "refs/tags/deploy-*") if l]
deploys = [(name, datetime.fromisoformat(when)) for name, when in tags]

lead_hours, failed, restore_hours, prev = [], 0, [], None
for i, (name, at) in enumerate(deploys):
    span = f"{prev}..{name}" if prev else name
    for line in filter(None, git("log", "--format=%cI %s", span)):
        when, subject = line.split(" ", 1)
        lead_hours.append((at - datetime.fromisoformat(when)).total_seconds() / 3600)
        if subject.startswith("hotfix:"):
            failed += 1
            restore_hours.append((at - deploys[i - 1][1]).total_seconds() / 3600)
    prev = name

days = (deploys[-1][1] - deploys[0][1]).days + 1
print(f"deploys:               {len(deploys)} in {days} days ({len(deploys) / days * 7:.1f} a week)")
print(f"lead time for changes: median {statistics.median(lead_hours):.1f} h")
print(f"change failure rate:   {failed}/{len(deploys) - failed} = {failed / (len(deploys) - failed):.0%}")
print(f"time to restore:       median {statistics.median(restore_hours):.1f} h")

$ python3 dora.py
deploys:               9 in 11 days (5.7 a week)
lead time for changes: median 3.0 h
change failure rate:   2/7 = 29%
time to restore:       median 1.4 h