manticoresearch/.github/workflows/clt_tests.yml

name: CLT tests 
on:
  workflow_call:
    inputs:
      docker_image:
        required: true
        type: string
        description: "Docker image to use for tests"
      artifact_name:
        required: false
        type: string
        description: "Name of the docker image artifact"
      repository:
        required: false
        type: string
        description: "Repository to checkout"
      ref:
        required: false
        type: string
        description: "Ref to checkout"
      continue_on_error:
        required: false
        type: boolean
        description: "Continue on error"
        default: false
    secrets:
      OPENAI_API_KEY:
        required: true
        description: "OpenAI API key for CLT tests"
      VOYAGE_API_KEY:
        required: true
        description: "Voyage API key for CLT tests"
      JINA_API_KEY:
        required: true
        description: "Jina API key for CLT tests"

jobs:
  discover:
    name: Discover CLT tests
    runs-on: ubuntu-22.04
    outputs:
      matrix: ${{ steps.build.outputs.matrix }}
    steps:
      - uses: actions/checkout@v4
        with:
          repository: ${{ inputs.repository }}
          ref: ${{ inputs.ref }}
      - id: build
        name: Build matrix
        shell: bash
        run: |
          set -euo pipefail
          python3 - <<'PY'
          import json
          import os
          import pathlib

          roots = [
              "test/clt-tests/buddy",
              "test/clt-tests/buddy-plugins",
              "test/clt-tests/bugs",
              "test/clt-tests/core",
              "test/clt-tests/data-manipulation",
              "test/clt-tests/expected-errors",
              "test/clt-tests/fulltext-search",
              "test/clt-tests/mcl",
              "test/clt-tests/http-interface",
              "test/clt-tests/indexer",
              "test/clt-tests/join",
              "test/clt-tests/kibana",
              "test/clt-tests/mysqldump/mysql",
              "test/clt-tests/mysqldump/maria",
              "test/clt-tests/performance",
              "test/clt-tests/replication",
              "test/clt-tests/sharding/cluster",
              "test/clt-tests/sharding/functional",
              "test/clt-tests/sharding/replication",
              "test/clt-tests/sharding/syntax",
              "test/clt-tests/test-configuration",
              "test/clt-tests/vector-knn",
          ]

          tests = []
          for root in roots:
              root_path = pathlib.Path(root)
              if not root_path.exists():
                  continue
              for path in sorted(root_path.glob("*.rec")):
                  # clt appends "*.rec" to the prefix; strip the suffix to avoid ".rec*.rec"
                  tests.append(str(path.with_suffix("")))

          total_jobs = 20
          if not tests:
              matrix = {"chunk": []}
          else:
              size = (len(tests) + total_jobs - 1) // total_jobs
              chunks = []
              for i in range(0, len(tests), size):
                  group = tests[i : i + size]
                  chunks.append(
                      {
                          "id": len(chunks),
                          "tests": group,
                          "test_prefix": "\n".join(group),
                      }
                  )
              matrix = {"chunk": chunks}

          print(f"Found {len(tests)} .rec tests across {len(matrix['chunk'])} jobs")
          output_path = os.environ["GITHUB_OUTPUT"]
          with open(output_path, "a", encoding="utf-8") as fh:
              fh.write(f"matrix={json.dumps(matrix)}\n")
          PY

  clt:
    name: CLT
    needs: [discover]
    runs-on: ubuntu-22.04
    timeout-minutes: 30
    continue-on-error: ${{ inputs.continue_on_error }}
    env:
      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
      VOYAGE_API_KEY: ${{ secrets.VOYAGE_API_KEY }}
      JINA_API_KEY: ${{ secrets.JINA_API_KEY }}
    strategy:
      fail-fast: false
      matrix: ${{ fromJson(needs.discover.outputs.matrix) }}
              
    steps:
      - uses: manticoresoftware/[email protected]
        with:
          artifact: ${{ inputs.artifact_name }}
          image: ${{ inputs.docker_image }}
          repository: ${{ inputs.repository }}
          ref: ${{ inputs.ref }}
          test_prefix: ${{ matrix.chunk.test_prefix }}
          comment_mode: failures
          run_args: "-e OPENAI_API_KEY -e VOYAGE_API_KEY -e JINA_API_KEY"
          ui_host: "https://clt.manticoresearch.com"