Browse Source

[mod] CI: refactor data-update.yml - searxng_extra/update scripts (#4732)

Style changes, cleanup and improved integration with CI by leveraging the use of
shared cache between all workflows (not functional until all workflows have been
refactored).
Ivan Gabaldon 5 days ago
parent
commit
c6a70782b2
1 changed files with 47 additions and 35 deletions
  1. 47 35
      .github/workflows/data-update.yml

+ 47 - 35
.github/workflows/data-update.yml

@@ -1,14 +1,27 @@
-name: "Update searx.data"
-on:  # yamllint disable-line rule:truthy
+---
+name: Update searx.data
+
+# yamllint disable-line rule:truthy
+on:
+  workflow_dispatch:
   schedule:
     - cron: "59 23 28 * *"
-  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref_name }}
+  cancel-in-progress: false
+
+permissions:
+  contents: read
+
+env:
+  PYTHON_VERSION: "3.13"
 
 jobs:
-  updateData:
-    name: Update data - ${{ matrix.fetch }}
-    runs-on: ubuntu-24.04
-    if: ${{ github.repository_owner == 'searxng'}}
+  data:
+    if: github.repository_owner == 'searxng'
+    name: ${{ matrix.fetch }}
+    runs-on: ubuntu-24.04-arm
     strategy:
       fail-fast: false
       matrix:
@@ -20,48 +33,47 @@ jobs:
           - update_engine_traits.py
           - update_wikidata_units.py
           - update_engine_descriptions.py
+
     steps:
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "${{ env.PYTHON_VERSION }}"
+
       - name: Checkout
         uses: actions/checkout@v4
+        with:
+          persist-credentials: "false"
 
-      - name: Install Ubuntu packages
-        run: |
-          sudo ./utils/searxng.sh install packages
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
+      - name: Setup cache Python
+        uses: actions/cache@v4
         with:
-          python-version: '3.12'
-          architecture: 'x64'
+          key: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-${{ hashFiles('./requirements*.txt') }}"
+          restore-keys: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-"
+          path: "./local/"
 
-      - name: Install Python dependencies
-        run: |
-          make V=1 install
+      - name: Setup venv
+        run: make V=1 install
 
       - name: Fetch data
-        env:
-          FETCH_SCRIPT: ./searxng_extra/update/${{ matrix.fetch }}
-        run: |
-          V=1 ./manage pyenv.cmd python "$FETCH_SCRIPT"
+        run: V=1 ./manage pyenv.cmd python "./searxng_extra/update/${{ matrix.fetch }}"
 
-      - name: Create Pull Request
+      - name: Create PR
         id: cpr
-        uses: peter-evans/create-pull-request@v6
+        uses: peter-evans/create-pull-request@v7
         with:
-          commit-message: '[data] update searx.data - ${{ matrix.fetch }}'
-          committer: searxng-bot <noreply@github.com>
-          author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>
-          signoff: false
-          branch: update_data_${{ matrix.fetch }}
-          delete-branch: true
-          draft: false
-          title: '[data] update searx.data - ${{ matrix.fetch }}'
-          body: |
-            update searx.data - ${{ matrix.fetch }}
+          author: "${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>"
+          committer: "searxng-bot <searxng-bot@users.noreply.github.com>"
+          title: "[data] update searx.data - ${{ matrix.fetch }}"
+          commit-message: "[data] update searx.data - ${{ matrix.fetch }}"
+          branch: "update_data_${{ matrix.fetch }}"
+          delete-branch: "true"
+          draft: "false"
+          signoff: "false"
           labels: |
             data
 
-      - name: Check outputs
+      - name: Display information
         run: |
           echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}"
           echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}"