BREAKING CHANGE: major release on 2026-03-26 includes a migration from elastic search to open search

.env.example

@@ -19,6 +19,11 @@ ACCESS_GROUP_SERVICE_HANDLER=""
 ACCESS_GROUPS_STATIC_ENABLED=true
 ACCESS_GROUPS_STATIC_VALUES=""
 ACCESS_GROUPS_OIDCPAYLOAD_ENABLED=true
+ACCESS_GROUPS_REST_ENABLED=true
+ACCESS_GROUPS_SERVICE_REST_AUTH_KEY="Authorization"
+ACCESS_GROUPS_SERVICE_REST_AUTH_VALUE=""
+ACCESS_GROUPS_SERVICE_REST_API_URL=""
+ACCESS_GROUPS_SERVICE_REST_USER_ID_FIELD="" #field to take as userID from user payload
 DOI_PREFIX="<DOI_PREFIX>"
 EXPRESS_SESSION_SECRET="<EXPRESS_SESSION_SECRET>"
 EXPRESS_SESSION_STORE=""
@@ -33,6 +38,7 @@ LDAP_SEARCH_BASE=<SEARCH_BASE>
 LDAP_SEARCH_FILTER="(LDAPUsername={{username}})"
 OIDC_ISSUER=""
 OIDC_CLIENT_ID=""
+OIDC_ADDITIONAL_AUTHORIZED_PARTIES="client1, client2"
 OIDC_CLIENT_SECRET=""
 OIDC_CALLBACK_URL=""
 OIDC_SCOPE=""
@@ -84,19 +90,17 @@ MS365_CLIENT_ID=<clientId>
 MS365_CLIENT_SECRET=<clientSecret>
 POLICY_PUBLICATION_SHIFT=3 years
 POLICY_RETENTION_SHIFT=-1 (indefinitely)
-ELASTICSEARCH_ENABLED=<"yes"|"no">
-ES_HOST="https://localhost:9200"
-ES_USERNAME="elastic"
-ES_PASSWORD="duo-password"
-ES_PORT=9200
-MONGODB_COLLECTION="Dataset"
-ES_MAX_RESULT=100000
-ES_FIELDS_LIMIT=400000
-ES_INDEX="dataset"
-ES_REFRESH=<"wait_for"|"false">
-STACK_VERSION="8.8.2"
-CLUSTER_NAME="es-cluster"
-MEM_LIMIT="4G"
+
+# "wait_for": waits for index refresh before returning, recommended for development and testing.
+# "false": returns immediately without waiting, recommended for production.
+OPENSEARCH_REFRESH=<"wait_for"|"false">
+OPENSEARCH_ENABLED=<"yes"|"no">
+OPENSEARCH_ENABLED="https://localhost:9200"
+OPENSEARCH_ENABLED="admin"
+OPENSEARCH_ENABLED="Scicat_default_password_2026"
+OPENSEARCH_DEFAULT_INDEX="dataset"
+OPENSEARCH_DATA_SYNC_BATCH_SIZE=50000
+
 FRONTEND_CONFIG_FILE="./src/config/frontend.config.json"
 FRONTEND_THEME_FILE="./src/config/frontend.theme.json"
 LOGGERS_CONFIG_FILE="loggers.json"

.github/workflows/auto-merge-dependabot.yml

@@ -15,7 +15,7 @@ jobs:
       ## Extract information about the dependencies being updated by a Dependabot-generated PR
       - name: Dependabot metadata
         id: dependabot-metadata
-        uses: dependabot/fetch-metadata@v2.5.0
+        uses: dependabot/fetch-metadata@v3.0.0
         with:
           github-token: "${{ secrets.GITHUB_TOKEN }}"
 

.github/workflows/deploy.yml

@@ -14,37 +14,37 @@ jobs:
         uses: actions/checkout@v6
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
+        uses: docker/setup-qemu-action@v4
         with:
           platforms: arm64
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@v4
 
       - name: Login to GHCR
-        uses: docker/login-action@v3
+        uses: docker/login-action@v4
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Create image tags
         id: meta
-        uses: docker/metadata-action@v5
+        uses: docker/metadata-action@v6
         with:
           images: ghcr.io/scicatproject/backend-next
           flavor: latest=true # adds :latest tag to outputs.tags
           tags: type=sha,format=long,prefix= # adds :<sha> tag to outputs.tags
 
       - name: Build and push
-        uses: docker/build-push-action@v6
+        uses: docker/build-push-action@v7
         with:
           context: .
           platforms: linux/amd64,linux/arm64/v8
           push: true
           tags: ${{ steps.meta.outputs.tags }}
 
-      - uses: actions/create-github-app-token@v2
+      - uses: actions/create-github-app-token@v3
         id: app-token
         with:
           app-id: ${{ secrets.WORKFLOW_DISPATCHER_APP_ID }}

.github/workflows/release-and-publish-sdk.yml

@@ -50,13 +50,13 @@ jobs:
       ## The setup-qemu-action simplifies the setup of QEMU for cross-platform builds
       ## https://github.com/docker/setup-qemu-action
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
+        uses: docker/setup-qemu-action@v4
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@v4
 
       - name: Login to GitHub Container Registry
-        uses: docker/login-action@v3
+        uses: docker/login-action@v4
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
@@ -75,7 +75,7 @@ jobs:
       ## https://github.com/docker/metadata-action
       - name: Docker meta
         id: meta
-        uses: docker/metadata-action@v5
+        uses: docker/metadata-action@v6
         with:
           images: ghcr.io/scicatproject/backend-next
           tags: |
@@ -85,14 +85,14 @@ jobs:
             type=raw,value={{date 'YYYY_MM'}},prefix=r_
 
       - name: Build and push
-        uses: docker/build-push-action@v6
+        uses: docker/build-push-action@v7
         with:
           context: .
           platforms: linux/amd64,linux/arm64/v8
           push: true
           tags: ${{ steps.meta.outputs.tags }}
 
-      - uses: actions/create-github-app-token@v2
+      - uses: actions/create-github-app-token@v3
         id: app-token
         with:
           app-id: ${{ secrets.WORKFLOW_DISPATCHER_APP_ID }}
@@ -146,7 +146,7 @@ jobs:
       - name: Download the Swagger schema
         run: curl -o ./swagger-schema.json http://localhost:3000/explorer-json
 
-      - uses: actions/upload-artifact@v6
+      - uses: actions/upload-artifact@v7
         with:
           name: swagger-schema-${{ github.sha }}
           path: ./swagger-schema.json
@@ -164,7 +164,7 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v6
 
-      - uses: actions/download-artifact@v7
+      - uses: actions/download-artifact@v8
         with:
           name: swagger-schema-${{ github.sha }}
           path: .
@@ -187,7 +187,7 @@ jobs:
             fi
             )
 
-      - uses: actions/upload-artifact@v6
+      - uses: actions/upload-artifact@v7
         with:
           name: sdk-${{ matrix.generator }}-${{ github.sha }}
           path: ./sdk
@@ -213,7 +213,7 @@ jobs:
           registry-url: "https://registry.npmjs.org/"
 
       - name: Download TypeScript SDK Artifact
-        uses: actions/download-artifact@v7
+        uses: actions/download-artifact@v8
         with:
           name: sdk-${{ matrix.sdk_type }}-${{ github.sha }}
           path: ./sdk
@@ -253,7 +253,7 @@ jobs:
           python-version: ${{ env.PYTHON_VERSION }}
 
       - name: Download Python SDK Artifact
-        uses: actions/download-artifact@v7
+        uses: actions/download-artifact@v8
         with:
           name: sdk-${{ matrix.sdk_type }}-${{ github.sha }}
           path: ./sdk

.github/workflows/test.yml

@@ -103,14 +103,14 @@ jobs:
         run: npm run build
 
   api_tests:
-    name: API tests with ElasticSearch enabled - ${{ matrix.elasticsearch_enabled }}
+    name: API tests with OpenSearch enabled - ${{ matrix.opensearch_enabled }}
     needs: [install-and-cache]
     runs-on: ubuntu-latest
 
     strategy:
       fail-fast: false
       matrix:
-        elasticsearch_enabled: ["yes", "no"]
+        opensearch_enabled: ["yes", "no"]
 
     steps:
       - name: Checkout code
@@ -130,13 +130,17 @@ jobs:
 
       - name: API tests
         env:
-          ELASTICSEARCH_ENABLED: ${{ matrix.elasticsearch_enabled }}
-          CLUSTER_NAME: es-cluster
-          ES_PASSWORD: duo-password
-          MEM_LIMIT: 4G
-          STACK_VERSION: 8.8.2
+          COMPOSE_PROFILES: ${{ matrix.opensearch_enabled == 'yes' && 'opensearch' || ''}}
+          OPENSEARCH_ENABLED: ${{ matrix.opensearch_enabled }}
+          OIDC_ISSUER: http://localhost:8080/local-test
+          OIDC_CLIENT_ID: scicat-client-test
+          OIDC_CLIENT_SECRET: secret
+          OIDC_SCOPE: openid profile email
+          OIDC_ADDITIONAL_AUTHORIZED_PARTIES: additional-authorized-client-test-1, additional-authorized-client-test-2
+
         # Start mongo container and app before running api tests
         run: |
+          cp opensearchConfig.example.json opensearchConfig.json
           cp CI/ESS/docker-compose.api.yaml docker-compose.yaml
-          docker compose up --build -d
-          npm run test:api
+          docker compose up --build -d --wait
+          OPENSEARCH_ENABLED=${{ matrix.opensearch_enabled }} npm run test:api

.github/workflows/upload-sdk-artifact.yml

@@ -43,7 +43,7 @@ jobs:
       - name: Download the Swagger schema
         run: curl -o ./swagger-schema.json http://localhost:3000/explorer-json
 
-      - uses: actions/upload-artifact@v6
+      - uses: actions/upload-artifact@v7
         with:
           name: swagger-schema-${{ github.sha }}
           path: ./swagger-schema.json
@@ -60,7 +60,7 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v6
 
-      - uses: actions/download-artifact@v7
+      - uses: actions/download-artifact@v8
         with:
           name: swagger-schema-${{ github.sha }}
           path: .
@@ -81,7 +81,7 @@ jobs:
             fi
             )
 
-      - uses: actions/upload-artifact@v6
+      - uses: actions/upload-artifact@v7
         if: github.event_name == 'push'
         with:
           name: sdk-${{ matrix.generator }}-${{ github.sha }}
@@ -93,6 +93,6 @@ jobs:
       - generate-and-upload-sdk
     if: github.event_name != 'push'
     steps:
-      - uses: geekyeggo/delete-artifact@v5
+      - uses: geekyeggo/delete-artifact@v6
         with:
           name: swagger-*

.gitignore

@@ -9,6 +9,7 @@ jobConfig.json
 jobConfig.yaml
 metricsConfig.json
 publishedDataConfig.json
+openSearchConfig.json
 
 # Configs
 .env

CI/E2E/.env.backend-next.example

@@ -39,12 +39,9 @@ DATASET_CREATION_VALIDATION_REGEX="^[0-9A-F]{8}-[0-9A-F]{4}-4[0-9A-F]{3}-[89AB][
 PROPOSAL_GROUPS="proposalingestor"
 SAMPLE_GROUPS=""
 
-ELASTICSEARCH_ENABLED='yes'
-ES_HOST=http://es01:9200
-ES_USERNAME=elastic
-ES_PASSWORD=duo-password
-MONGODB_COLLECTION=Dataset
-ES_MAX_RESULT=210000
-ES_FIELDS_LIMIT=400000
-ES_INDEX="dataset"
-ES_REFRESH="wait_for"
+OPENSEARCH_ENABLED='yes'
+OPENSEARCH_DEFAULT_INDEX="dataset"
+OPENSEARCH_HOST=https://localhost:9200
+OPENSEARCH_USERNAME=admin
+OPENSEARCH_PASSWORD=Scicat_default_password_2026
+OPENSEARCH_REFRESH="wait_for"

CI/E2E/backend.env

@@ -36,12 +36,9 @@ DATASET_CREATION_VALIDATION_REGEX="^[0-9A-F]{8}-[0-9A-F]{4}-4[0-9A-F]{3}-[89AB][
 PROPOSAL_GROUPS="proposalingestor"
 SAMPLE_GROUPS=""
 
-ELASTICSEARCH_ENABLED='yes'
-ES_HOST=http://es01:9200
-ES_USERNAME=elastic
-ES_PASSWORD=duo-password
-MONGODB_COLLECTION=Dataset
-ES_MAX_RESULT=210000
-ES_FIELDS_LIMIT=400000
-ES_INDEX="dataset"
-ES_REFRESH="wait_for"
+OPENSEARCH_ENABLED=yes
+OPENSEARCH_DEFAULT_INDEX="dataset"
+OPENSEARCH_HOST=https://localhost:9200
+OPENSEARCH_USERNAME=admin
+OPENSEARCH_PASSWORD=Scicat_default_password_2026
+OPENSEARCH_REFRESH="wait_for"