Add autoupdate (#3067)

* add autoupdate.lua

* fetch releases list

* check tar shasum

* get shasum of tags

* update version

* fix current branch

* patch version

* create pr

* fix pr

* add base and head

* fix error

* add verbose

* push commit

* add autoupdate ci

.github/workflows/autoupdate.yml

@@ -0,0 +1,43 @@
+name: Auto-update packages
+
+on:
+  schedule: # execute every 24 hours
+    - cron: "0 */24 * * *"
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+
+    runs-on: ${{ matrix.os }}
+
+    steps:
+      - uses: actions/checkout@v1
+      - uses: xmake-io/github-action-setup-xmake@v1
+        with:
+          xmake-version: latest
+          actions-cache-folder: '.xmake-cache'
+
+      - name: Installation
+        run: |
+          curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg
+          echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null
+          sudo apt update
+          sudo apt install -y gh unzip zip
+          echo ${{ secrets.GITHUB_TOKEN }} | gh auth login --with-token
+          git config --global user.email "[email protected]"
+          git config --global user.name "ruki"
+
+      - name: Install SSH key
+        uses: shimataro/ssh-key-action@v2
+        with:
+          key: ${{ secrets.SSH_KEY }}
+          name: id_rsa # optional
+          known_hosts: ${{ secrets.KNOWN_HOSTS }}
+
+      - name: Auto-update packages
+        run: |
+          xmake l -vD scripts/autoupdate.lua 1
+
+

scripts/autoupdate.lua

@@ -0,0 +1,93 @@
+import("core.package.package")
+import("core.base.semver")
+import("core.base.hashset")
+import("packages", {alias = "packages_util"})
+
+function _load_package(packagename, packagedir, packagefile)
+    local funcinfo = debug.getinfo(package.load_from_repository)
+    if funcinfo and funcinfo.nparams == 3 then -- >= 2.7.8
+        return package.load_from_repository(packagename, packagedir, {packagefile = packagefile})
+    else
+        -- deprecated
+        return package.load_from_repository(packagename, nil, packagedir, packagefile)
+    end
+end
+
+function _get_all_packages()
+    local packages = _g.packages
+    if not packages then
+        packages = {}
+        for _, packagedir in ipairs(os.dirs(path.join("packages", "*", "*"))) do
+            local packagename = path.filename(packagedir)
+            local packagefile = path.join(packagedir, "xmake.lua")
+            local instance = _load_package(packagename, packagedir, packagefile)
+            local basename = instance:get("base")
+            if instance and basename then
+                local basedir = path.join("packages", basename:sub(1, 1):lower(), basename:lower())
+                local basefile = path.join(basedir, "xmake.lua")
+                instance._BASE = _load_package(basename, basedir, basefile)
+            end
+            if instance then
+                table.insert(packages, instance)
+            end
+        end
+        _g.packages = packages
+    end
+    return packages
+end
+
+function _update_version(instance, version, shasum)
+    local branch = "autoupdate-" .. instance:name() .. "-" .. version
+    local branch_current = os.iorun("git branch --show-current"):trim()
+    local repourl = "https://github.com/xmake-io/xmake-repo.git"
+    os.vexec("git stash")
+    os.vexec("git branch -D %s", branch)
+    os.vexec("git checkout dev")
+    os.vexec("git pull %s dev", repourl)
+    os.vexec("git branch %s", branch)
+    os.vexec("git checkout %s", branch)
+    local scriptfile = path.join(instance:scriptdir(), "xmake.lua")
+    if os.isfile(scriptfile) then
+        local inserted = false
+        local version_current
+        io.gsub(scriptfile, "add_versions%(\"(.-)\",%s+\"(.-)\"%)", function (v, h)
+            if not version_current or sermver.compare(v, version_current) > 0 then
+                version_current = v
+            end
+            if not inserted then
+                inserted = true
+                return string.format('add_versions("%s", "%s")\n    add_versions("%s", "%s")', version, shasum, v, h)
+            end
+        end)
+        local body = string.format("New version of %s detected (package version: %s, last github version: %s)",
+            instance:name(), version_current, version)
+        os.vexec("git add .")
+        os.vexec("git commit -a -m \"Update %s to %s\"", instance:name(), version)
+        os.vexec("git push %s %s:%s", repourl, branch, branch)
+        os.vexec("gh pr create --title \"Auto-update %s to %s\" --body \"%s\" -R xmake-io/xmake-repo -B dev -H %s", instance:name(), version, body, branch)
+    end
+    os.vexec("git reset --hard HEAD")
+    os.vexec("git checkout %s", branch_current)
+    os.vexec("git stash pop")
+end
+
+function main(maxcount)
+    local count = 0
+    local maxcount = tonumber(maxcount or 10)
+    local instances = _get_all_packages()
+    for _, instance in ipairs(instances) do
+        local checkupdate_filepath = path.join(instance:scriptdir(), "checkupdate.lua")
+        if not os.isfile(checkupdate_filepath) then
+            checkupdate_filepath = path.join(os.scriptdir(), "checkupdate.lua")
+        end
+        if os.isfile(checkupdate_filepath) and count < maxcount then
+            local checkupdate = import("checkupdate", {rootdir = path.directory(checkupdate_filepath), anonymous = true})
+            local version, shasum = checkupdate(instance)
+            if version and shasum then
+                cprint("package(%s): new version ${bright}%s${clear} found, shasum: ${bright}%s", instance:name(), version, shasum)
+                _update_version(instance, version, shasum)
+                count = count + 1
+            end
+        end
+    end
+end

scripts/checkupdate.lua

@@ -0,0 +1,113 @@
+import("core.base.semver")
+import("net.http")
+import("devel.git")
+import("private.action.require.impl.utils.filter")
+
+function shasum_of(package, url, version)
+    local shasum
+    local tmpfile = os.tmpfile()
+    package:version_set(version)
+    url = filter.handle(url, package)
+    local ok = try { function() http.download(url, tmpfile); return true end }
+    if ok and os.isfile(tmpfile) and os.filesize(tmpfile) > 1024 then
+        shasum = hash.sha256(tmpfile)
+    end
+    os.tryrm(tmpfile)
+    return shasum
+end
+
+function _is_valid_version(version)
+    if not semver.is_valid(version) then
+        return false
+    end
+    local v = semver.new(version)
+    local prerelease = v:prerelease()
+    if prerelease and #prerelease > 0 then
+        return false
+    end
+    return true
+end
+
+function _get_version_and_shasum(package, url, version_latest)
+    if version_latest then
+        local has_prefix_v = false
+        for _, version in ipairs(package:versions()) do
+            if version:startswith("v") then
+                has_prefix_v = true
+            end
+            if semver.compare(version, version_latest) >= 0 then
+                version_latest = nil
+                break
+            end
+        end
+        if version_latest then
+            if has_prefix_v and not version_latest:startswith("v") then
+                version_latest = "v" .. version_latest
+            elseif not has_prefix_v and version_latest:startswith("v") then
+                version_latest = version_latest:sub(2)
+            end
+        end
+    end
+    if version_latest then
+        local shasum = shasum_of(package, url, version_latest)
+        if shasum then
+            return version_latest, shasum
+        end
+    end
+end
+
+function _check_version_from_github_tags(package, url)
+    local repourl = url:match("https://github%.com/.-/.-/")
+    if repourl then
+        print("checking version from github tags %s ..", repourl)
+        local version_latest
+        local tags = git.tags(repourl)
+        for _, tag in ipairs(tags) do
+            if _is_valid_version(tag) and (not version_latest or semver.compare(tag, version_latest) > 0) then
+                version_latest = tag
+            end
+        end
+        if version_latest then
+            return _get_version_and_shasum(package, url, version_latest)
+        end
+    end
+end
+
+function _check_version_from_github_releases(package, url)
+    local repourl = url:match("https://github%.com/.-/.-/")
+    if repourl then
+        print("checking version from github releases %s ..", repourl)
+        local list = try {function() return os.iorunv("gh", {"release", "list", "--exclude-drafts", "--exclude-pre-releases", "-R", repourl}) end}
+        if not list then
+            list = os.iorunv("gh", {"release", "list", "-R", repourl})
+        end
+        if list then
+            local version_latest
+            for _, line in ipairs(list:split("\n")) do
+                local splitinfo = line:split("%s+")
+                local version = splitinfo[#splitinfo - 1]
+                if version and _is_valid_version(version) then
+                    version_latest = version
+                    break
+                end
+            end
+            if version_latest then
+                return _get_version_and_shasum(package, url, version_latest)
+            end
+        end
+    end
+end
+
+function main(package)
+    local checkers = {
+        ["https://github%.com/.-/.-/archive/refs/tags/.*"] = _check_version_from_github_tags,
+        ["https://github%.com/.-/.-/releases/download/.*"] = _check_version_from_github_releases
+    }
+    for _, url in ipairs(package:urls()) do
+        for pattern, checker in pairs(checkers) do
+            if url:match(pattern) then
+                return checker(package, url)
+            end
+        end
+    end
+end