Changes in Github CI:

- no self-hosted runners at all
- use of an action to publish
- other minor changes

.github/workflows/clt.yml

@@ -7,6 +7,11 @@ on:
   pull_request:
     branches: [ master ]
 
+# cancels the previous workflow run when a new one appears in the same branch (e.g. master or a PR's branch)
+concurrency:
+  group: clt_${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   rhel-release:
     if: github.event_name == 'workflow_dispatch'

.github/workflows/download_artifact_with_retries/action.yml

@@ -1,52 +0,0 @@
-name: 'Download artifact with retries'
-description: 'This action makes a few attempts to download an artifact'
-inputs:
-  name:
-    required: true
-    type: string
-  path:
-    required: true
-    type: string
-runs:
-  using: "composite"
-  steps:
-    - name: Download - 1st attempt
-      id: download1
-      uses: actions/download-artifact@v3
-      with:
-        name: ${{ inputs.name }}
-        path: ${{ inputs.path }}
-      continue-on-error: true
-    - name: Download - 2nd attempt
-      id: download2
-      if: steps.download1.outcome == 'failure'
-      uses: actions/download-artifact@v3
-      with:
-        name: ${{ inputs.name }}
-        path: ${{ inputs.path }}
-      continue-on-error: true
-    - name: Download - 3rd attempt
-      id: download3
-      if: steps.download2.outcome == 'failure'
-      uses: actions/download-artifact@v3
-      with:
-        name: ${{ inputs.name }}
-        path: ${{ inputs.path }}
-      continue-on-error: true
-    - name: Check if any download was successful
-      if: success() || failure()
-      run: |
-        if [ "${{ steps.download1.outcome }}" == "success" ] || [ "${{ steps.download2.outcome }}" == "success" ] || [ "${{ steps.download3.outcome }}" == "success" ]; then
-          echo "Download was successful"
-        else
-          echo "All download attempts failed"
-          exit 1
-        fi
-      shell: bash
-    - name: Untar
-      if: success() || failure()
-      run: |
-        ls -la ${{ inputs.path }}/artifact.tar
-        tar -xvf ${{ inputs.path }}/artifact.tar
-        ls -la
-      shell: bash

.github/workflows/pack.yml → .github/workflows/pack_publish.yml

@@ -117,82 +117,74 @@ jobs:
       - run: echo "Ready to publish"
 
   publish_debian_ubuntu:
+    name: Publishing Debian/Ubuntu
     needs: publish
-    runs-on: [self-hosted, Linux, X64]
     strategy:
       fail-fast: true
       matrix:
         DISTR: [bionic, focal, jammy, buster, bullseye, bookworm]
         arch: [x86_64, aarch64]
-    name: ${{ matrix.DISTR }} ${{ matrix.arch }} publishing
+    runs-on: ubuntu-22.04
     steps:
-      - uses: manticoresoftware/download_artifact_with_retries@main
+      - uses: manticoresoftware/publish_to_repo@main
         with:
-          name: build_${{ matrix.DISTR }}_RelWithDebInfo_${{ matrix.arch }}
-          path: .
-      - name: Deploy package
-        run: |
-          curl -sSL https://raw.githubusercontent.com/manticoresoftware/repo_scripts/main/upload_repo_deb > script
-          chmod +x script
-          DISTRO=${{ matrix.DISTR }} DIGIT_DELIMITER2="-" ./script
-        shell: bash
+          ssh_key: ${{ secrets.REPO_SSH_KEY }}
+          distr: ${{ matrix.DISTR }}
+          arch: ${{ matrix.arch }}
+          artifact: build_${{ matrix.DISTR }}_RelWithDebInfo_${{ matrix.arch }}
+          type: deb
+          delimiter: "-"
 
   publish_rhel:
+    name: Publishing RHEL
     needs: publish
-    runs-on: [self-hosted, Linux, X64]
     strategy:
       fail-fast: true
       matrix:
         DISTR: [7, 8, 9]
         arch: [x86_64, aarch64]
-    name: RHEL ${{ matrix.DISTR }} ${{ matrix.arch }} publishing
+    runs-on: ubuntu-22.04
     steps:
-      - uses: manticoresoftware/download_artifact_with_retries@main
+      - uses: manticoresoftware/publish_to_repo@main
         with:
-          name: build_rhel${{ matrix.DISTR }}_RelWithDebInfo_${{ matrix.arch }}
-          path: .
-      - name: Deploy package
-        run: |
-          curl -sSL https://raw.githubusercontent.com/manticoresoftware/repo_scripts/main/upload_repo_rpm > script
-          chmod +x script
-          DISTRO=${{ matrix.DISTR }} DIGIT_DELIMITER2="_" ./script
-        shell: bash
+          ssh_key: ${{ secrets.REPO_SSH_KEY }}
+          distr: ${{ matrix.DISTR }}
+          arch: ${{ matrix.arch }}
+          artifact: build_rhel${{ matrix.DISTR }}_RelWithDebInfo_${{ matrix.arch }}
+          type: rpm
+          delimiter: "_"
 
   publish_macos:
+    name: Publishing MacOS
     needs: publish
-    runs-on: [self-hosted, Linux, X64]
     strategy:
       fail-fast: true
       matrix:
         arch: [x86_64, arm64]
-    name: macos ${{ matrix.arch }} publishing
+    runs-on: ubuntu-22.04
     steps:
-      - uses: manticoresoftware/download_artifact_with_retries@main
+      - uses: manticoresoftware/publish_to_repo@main
         with:
-          name: build_macos_RelWithDebInfo_${{ matrix.arch }}
-          path: .
-      - name: Deploy package
-        run: |
-          curl -sSL https://raw.githubusercontent.com/manticoresoftware/repo_scripts/main/upload_repo_arc > script
-          chmod +x script
-          DIGIT_DELIMITER2="-" ./script
-        shell: bash
+          ssh_key: ${{ secrets.REPO_SSH_KEY }}
+          distr: macos
+          arch: ${{ matrix.arch }}
+          artifact: build_macos_RelWithDebInfo_${{ matrix.arch }}
+          type: arc
+          delimiter: "-"
 
   publish_windows:
-    name: Publishing Windows packages to repo.manticoresearch.com
+    name: Publishing Windows
     needs: publish
-    runs-on: [self-hosted, Linux, X64]
+    runs-on: ubuntu-22.04
     steps:
-      - uses: manticoresoftware/download_artifact_with_retries@main
+      - uses: manticoresoftware/publish_to_repo@main
         with:
-          name: build_windows_RelWithDebInfo_x64
-          path: .
-      - name: Deploy package
-        run: |
-          curl -sSL https://raw.githubusercontent.com/manticoresoftware/repo_scripts/main/upload_repo_arc > script
-          chmod +x script
-          DIGIT_DELIMITER2="-" ./script
-        shell: bash
+          ssh_key: ${{ secrets.REPO_SSH_KEY }}
+          distr: windows
+          arch: x64
+          artifact: build_windows_RelWithDebInfo_x64
+          type: arc
+          delimiter: "-"
 
   build_nsis:
     name: Making and publishing Windows NSIS installer
@@ -224,20 +216,18 @@ jobs:
           path: build/manticore-*.exe
 
   publish_nsis:
-    name: Publishing Windows NSIS installer to repo.manticoresearch.com
-    runs-on: [self-hosted, Linux, X64]
+    name: Publishing Windows NSIS installer
     needs: build_nsis
+    runs-on: ubuntu-22.04
     steps:
-      - uses: manticoresoftware/download_artifact_with_retries@main
+      - uses: manticoresoftware/publish_to_repo@main
         with:
-          name: win_installer
-          path: .
-      - name: Deploy package
-        run: |
-          wget https://raw.githubusercontent.com/manticoresoftware/repo_scripts/main/upload_repo_arc
-          chmod +x upload_repo_arc
-          /bin/bash ./upload_repo_arc
-        shell: bash
+          ssh_key: ${{ secrets.REPO_SSH_KEY }}
+          distr:
+          arch:
+          artifact: win_installer
+          type: arc
+          delimiter:
 
   build_docker:
     name: Building and pushing docker
@@ -256,6 +246,5 @@ jobs:
       - name: Calculate short commit hash
         id: sha
         run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
-      # TODO: remove tmp from dockerhub_deploy.sh
       - name: Building docker
         run: CI_COMMIT_SHORT_SHA=${{ steps.sha.outputs.sha_short }} /bin/bash dist/dockerhub_deploy.sh

.github/workflows/test.yml

@@ -16,6 +16,9 @@ jobs:
 
   win_bundle:
     name: Windows supplementary files preparation
+    if: |
+      (github.event_name == 'push' && github.ref == 'refs/heads/master')
+      || (github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'pack') && !contains(github.event.pull_request.labels.*.name, 'publish'))
     runs-on: ubuntu-22.04
     steps:
       - name: Check out cache
@@ -42,12 +45,18 @@ jobs:
 
   build_linux_debug:
     name: Linux debug build
+    if: |
+      (github.event_name == 'push' && github.ref == 'refs/heads/master')
+      || (github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'pack') && !contains(github.event.pull_request.labels.*.name, 'publish'))
     uses: ./.github/workflows/build_template.yml
     with:
       CTEST_CONFIGURATION_TYPE: "Debug"
 
   test_linux_debug:
     name: Linux debug mode tests
+    if: |
+      (github.event_name == 'push' && github.ref == 'refs/heads/master')
+      || (github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'pack') && !contains(github.event.pull_request.labels.*.name, 'publish'))
     needs: build_linux_debug
     uses: ./.github/workflows/test_template.yml
     strategy:
@@ -95,6 +104,9 @@ jobs:
 
   debug_tests_report:
     name: Debug mode tests summary and report
+    if: |
+      (github.event_name == 'push' && github.ref == 'refs/heads/master')
+      || (github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'pack') && !contains(github.event.pull_request.labels.*.name, 'publish'))
     needs: test_linux_debug
     runs-on: ubuntu-22.04
     steps:
@@ -191,10 +203,16 @@ jobs:
 
   build_linux_release:
     name: Linux release build
+    if: |
+      (github.event_name == 'push' && github.ref == 'refs/heads/master')
+      || (github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'pack') && !contains(github.event.pull_request.labels.*.name, 'publish'))
     uses: ./.github/workflows/build_template.yml
 
   test_linux_release:
     name: Linux release mode tests
+    if: |
+      (github.event_name == 'push' && github.ref == 'refs/heads/master')
+      || (github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'pack') && !contains(github.event.pull_request.labels.*.name, 'publish'))
     needs: build_linux_release
     uses: ./.github/workflows/test_template.yml
     strategy:
@@ -242,6 +260,9 @@ jobs:
 
   release_tests_report:
     name: Release mode tests summary and report
+    if: |
+      (github.event_name == 'push' && github.ref == 'refs/heads/master')
+      || (github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'pack') && !contains(github.event.pull_request.labels.*.name, 'publish'))
     needs: test_linux_release
     runs-on: ubuntu-22.04
     steps:
@@ -338,6 +359,9 @@ jobs:
 
   pack_focal:
     name: Focal x86_64 package
+    if: |
+      (github.event_name == 'push' && github.ref == 'refs/heads/master')
+      || (github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'pack') && !contains(github.event.pull_request.labels.*.name, 'publish'))
     uses: ./.github/workflows/build_template.yml
     with:
       DISTR: focal
@@ -352,6 +376,9 @@ jobs:
 
   build_clt_docker:
     name: CLT docker image
+    if: |
+      (github.event_name == 'push' && github.ref == 'refs/heads/master')
+      || (github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'pack') && !contains(github.event.pull_request.labels.*.name, 'publish'))
     needs: pack_focal
     runs-on: ubuntu-22.04
     env:
@@ -387,6 +414,9 @@ jobs:
 
   build_aarch64:
     name: Linux aarch64 build
+    if: |
+      (github.event_name == 'push' && github.ref == 'refs/heads/master')
+      || (github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'pack') && !contains(github.event.pull_request.labels.*.name, 'publish'))
     uses: ./.github/workflows/build_template.yml
     with:
       arch: aarch64
@@ -395,6 +425,9 @@ jobs:
 
   build_freebsd:
     name: FreeBSD x86_64 build
+    if: |
+      (github.event_name == 'push' && github.ref == 'refs/heads/master')
+      || (github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'pack') && !contains(github.event.pull_request.labels.*.name, 'publish'))
     uses: ./.github/workflows/build_template.yml
     with:
       DISTR: freebsd13
@@ -404,6 +437,9 @@ jobs:
 
   build_windows:
     name: Windows x64 build
+    if: |
+      (github.event_name == 'push' && github.ref == 'refs/heads/master')
+      || (github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'pack') && !contains(github.event.pull_request.labels.*.name, 'publish'))
     uses: ./.github/workflows/build_template.yml
     with:
       DISTR: windows
@@ -416,6 +452,9 @@ jobs:
 
   test_windows:
     name: Windows tests
+    if: |
+      (github.event_name == 'push' && github.ref == 'refs/heads/master')
+      || (github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'pack') && !contains(github.event.pull_request.labels.*.name, 'publish'))
     needs: [build_windows, win_bundle]
     uses: ./.github/workflows/win_test_template.yml
     strategy:
@@ -439,6 +478,9 @@ jobs:
 
   windows_tests_report:
     name: Windows tests summary and report
+    if: |
+      (github.event_name == 'push' && github.ref == 'refs/heads/master')
+      || (github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'pack') && !contains(github.event.pull_request.labels.*.name, 'publish'))
     needs: test_windows
     runs-on: ubuntu-22.04
     container:

.github/workflows/upload_artifact_with_retries/action.yml

@@ -1,52 +0,0 @@
-name: 'Upload artifact with retries'
-description: 'This action makes a few attempts to upload an artifact'
-inputs:
-  name:
-    required: true
-    type: string
-  path:
-    required: true
-    type: string
-runs:
-  using: "composite"
-  steps:
-    - name: Tar # this step is important to preserve artifact permissions, without tar'ing they are lost
-      id: tar
-      run: |
-        shopt -s globstar
-        tar -cvf artifact.tar --ignore-failed-read ${{ inputs.path }}
-        ls -la artifact.tar
-      shell: bash
-    - name: Upload - 1st attempt
-      id: upload1
-      uses: cytopia/[email protected]
-      with:
-        name: ${{ inputs.name }}
-        path: artifact.tar
-      continue-on-error: true
-    - name: Upload build artifacts 2nd attempt
-      id: upload2
-      if: steps.upload1.outcome == 'failure'
-      uses: cytopia/[email protected]
-      with:
-        name: ${{ inputs.name }}
-        path: artifact.tar
-      continue-on-error: true
-    - name: Upload build artifacts 3rd attempt
-      id: upload3
-      if: steps.upload2.outcome == 'failure'
-      uses: cytopia/[email protected]
-      with:
-        name: ${{ inputs.name }}
-        path: artifact.tar
-      continue-on-error: true
-    - name: Check if any upload was successful
-      if: success() || failure()
-      run: |
-        if [ "${{ steps.upload1.outcome }}" == "success" ] || [ "${{ steps.upload2.outcome }}" == "success" ] || [ "${{ steps.upload3.outcome }}" == "success" ]; then
-          echo "Upload was successful"
-        else
-          echo "All upload attempts failed"
-          exit 1
-        fi
-      shell: bash

dist/dockerhub_deploy.sh

@@ -12,7 +12,7 @@ if [[ "$FAILED_TO_LOGIN" == 'true' ]]; then
 fi
 
 if [ ! -n "$BUILD_TAGS" ]; then
-  BUILD_TAGS="dev-tmp dev-tmp-$( cat src/sphinxversion.h.in | grep VERNUMBERS | cut -d" " -f3 | cut -d'"' -f2 )-$CI_COMMIT_SHORT_SHA"
+  BUILD_TAGS="dev dev-$( cat src/sphinxversion.h.in | grep VERNUMBERS | cut -d" " -f3 | cut -d'"' -f2 )-$CI_COMMIT_SHORT_SHA"
 fi
 
 IFS=' ' read -r -a SPLITTED_BUILD_TAGS <<<"$BUILD_TAGS"