diff --git a/.github/workflows/start-venv.yaml b/.github/workflows/start-venv.yaml index 3def95a..15c1be0 100644 --- a/.github/workflows/start-venv.yaml +++ b/.github/workflows/start-venv.yaml @@ -46,6 +46,16 @@ on: required: false description: 'Set to true in order to trigger slack notification' default: true + useVerdaccio: + type: boolean + required: false + description: 'Build Docker with custom @frontegg packages from a template-and-libs branch via local Verdaccio' + default: false + templateAndLibsBranch: + type: string + required: false + description: 'Branch of frontegg/template-and-libs to build and publish to local Verdaccio (only used when useVerdaccio is true)' + default: '' secrets: GH_REPOSITORY_ADMIN_TOKEN: description: 'Github repository admin token' @@ -118,6 +128,37 @@ jobs: const result = { commits: splitString.map(x => x.toString()) }; console.log(`result: ${JSON.stringify(result)}`); return JSON.stringify(result); + - name: Update venv repo appVersion for Verdaccio + if: ${{ inputs.useVerdaccio && steps.venv-actions.outputs.commits != '' }} + working-directory: ./venv + run: | + git config user.email "shadow@frontegg.com" + git config user.name "shadow-agent" + IFS=',' read -ra ENTRIES <<< "${{ steps.venv-actions.outputs.commits }}" + CHANGED=false + for entry in "${ENTRIES[@]}"; do + HASH="${entry##*:}" + SHORT_SHA="${HASH:0:7}" + OLD_APP_VERSION="venv-${SHORT_SHA}" + NEW_APP_VERSION="shadow-${{ github.run_id }}-${{ github.run_attempt }}-${SHORT_SHA}" + echo "Replacing appVersion: ${OLD_APP_VERSION} -> ${NEW_APP_VERSION}" + if grep -rl "appVersion: ${OLD_APP_VERSION}" environments/ 2>/dev/null; then + grep -rl "appVersion: ${OLD_APP_VERSION}" environments/ | xargs sed -i "s|appVersion: ${OLD_APP_VERSION}|appVersion: ${NEW_APP_VERSION}|g" + CHANGED=true + elif grep -rl "appVersion: \"${OLD_APP_VERSION}\"" environments/ 2>/dev/null; then + grep -rl "appVersion: \"${OLD_APP_VERSION}\"" environments/ | xargs sed -i "s|appVersion: \"${OLD_APP_VERSION}\"|appVersion: \"${NEW_APP_VERSION}\"|g" + CHANGED=true + else + echo "::warning::appVersion ${OLD_APP_VERSION} not found in venv environments" + fi + done + if [ "${CHANGED}" = "true" ]; then + git add . + git commit -m "venv: verdaccio appVersion for run ${{ github.run_id }}-${{ github.run_attempt }}" + git fetch origin master + git rebase origin/master + git push origin master + fi build-docker: name: Build services docker images for venv @@ -128,6 +169,11 @@ jobs: matrix: ${{ fromJSON(needs.start-venv.outputs.commits) }} runs-on: ubuntu-latest steps: + - name: Validate Verdaccio inputs + if: ${{ inputs.useVerdaccio && inputs.templateAndLibsBranch == '' }} + run: | + echo "::error::useVerdaccio is true but templateAndLibsBranch is empty" + exit 1 - name: Display Commit name ${{ matrix.commits }} run: | echo Building ${{ matrix.commits }} docker @@ -159,13 +205,187 @@ jobs: uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + with: + driver-opts: network=host - name: Login to Docker Hub uses: docker/login-action@v3 with: username: ${{ secrets.DOCKER_HUB_ACTION_USER }} password: ${{ secrets.DOCKER_HUB_ACTION_PASSWORD }} + - name: Write Verdaccio config + if: ${{ inputs.useVerdaccio }} + env: + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} + run: | + mkdir -p /tmp/verdaccio + cat > /tmp/verdaccio/config.yaml < /dev/null; then + git config --local --unset http.https://github.com/.extraheader + exit 0 + fi + git fetch --no-tags --deepen=50 origin "${REFSPEC_MASTER}" "${REFSPEC_BRANCH}" + done + + git config --local --unset-all http.https://github.com/.extraheader || true + echo "::error::Unable to find merge-base between ${TARGET_BRANCH} and master after deepening fetch history" + exit 1 + - name: Install library dependencies + if: ${{ inputs.useVerdaccio }} + working-directory: _shadow-libs + run: yarn install --immutable || YARN_ENABLE_IMMUTABLE_INSTALLS=false yarn install + env: + YARN_NPM_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + # yarn install may modify generated files; commit them so lerna list --since + # compares only real code changes against origin/master. + - name: Commit post-install state + if: ${{ inputs.useVerdaccio }} + working-directory: _shadow-libs + run: | + git config user.email "shadow@frontegg.com" + git config user.name "shadow-agent" + git add -A + if ! git diff --cached --exit-code; then + git commit -m "chore: post-install" + fi + - name: Detect changed packages + if: ${{ inputs.useVerdaccio }} + id: lerna-changed + working-directory: _shadow-libs + run: | + set +e + CHANGED_JSON=$(npx lerna list --since origin/master --all --exclude-dependents --json 2>lerna-list.stderr) + LERNA_EXIT=$? + set -e + if [ "${LERNA_EXIT}" -ne 0 ]; then + echo "::warning::lerna list --since exited ${LERNA_EXIT}; stderr:" + cat lerna-list.stderr || true + CHANGED_JSON="[]" + fi + echo "Changed packages: ${CHANGED_JSON}" + + if [ "${CHANGED_JSON}" = "[]" ]; then + echo "has_changes=false" >> "$GITHUB_OUTPUT" + else + echo "has_changes=true" >> "$GITHUB_OUTPUT" + NAMES=$(echo "${CHANGED_JSON}" | node -e " + const pkgs = JSON.parse(require('fs').readFileSync(0,'utf8')); + process.stdout.write(pkgs.map(p => p.name).join(',')); + ") + echo "names=${NAMES}" >> "$GITHUB_OUTPUT" + fi + - name: Build affected packages + if: ${{ inputs.useVerdaccio && steps.lerna-changed.outputs.has_changes == 'true' }} + working-directory: _shadow-libs + run: yarn build:ci + - name: Register Verdaccio user and token + if: ${{ inputs.useVerdaccio && steps.lerna-changed.outputs.has_changes == 'true' }} + run: | + REG_BODY="$(curl -sf -X PUT "http://127.0.0.1:4873/-/user/org.couchdb.user:shadow" \ + -H "Content-Type: application/json" \ + -d '{"name":"shadow","password":"shadow","email":"shadow@frontegg.com","type":"user"}')" + echo "VERDACCIO_NPM_TOKEN=$(echo "${REG_BODY}" | node -p "JSON.parse(require('fs').readFileSync(0,'utf8')).token")" >> "$GITHUB_ENV" + - name: Publish changed packages to Verdaccio + if: ${{ inputs.useVerdaccio && steps.lerna-changed.outputs.has_changes == 'true' }} + id: publish-shadow + working-directory: _shadow-libs + run: | + echo "//127.0.0.1:4873/:_authToken=${VERDACCIO_NPM_TOKEN}" >> .npmrc + + npx lerna version prerelease \ + --force-publish=${{ steps.lerna-changed.outputs.names }} \ + --no-push \ + --preid "venv.${{ github.run_id }}.${{ github.run_attempt }}" \ + --yes + + SHADOW_VERSION="$(node -p "require('./lerna.json').version")" + echo "published_version=${SHADOW_VERSION}" >> "$GITHUB_OUTPUT" + + npx lerna publish from-package \ + --registry http://127.0.0.1:4873 \ + --no-verify-access \ + --yes + - name: Verdaccio summary + if: ${{ inputs.useVerdaccio && steps.lerna-changed.outputs.has_changes == 'true' }} + run: | + echo "### Verdaccio Shadow Build" >> "$GITHUB_STEP_SUMMARY" + echo "**Version:** \`${{ steps.publish-shadow.outputs.published_version }}\`" >> "$GITHUB_STEP_SUMMARY" + echo "**Packages bumped in service:**" >> "$GITHUB_STEP_SUMMARY" + IFS=',' read -ra PKGS <<< "${{ steps.lerna-changed.outputs.names }}" + for pkg in "${PKGS[@]}"; do + echo "- \`${pkg}\`" >> "$GITHUB_STEP_SUMMARY" + done + - name: Clean up shadow libs from build context + if: ${{ inputs.useVerdaccio }} + run: rm -rf _shadow-libs + # Non-Verdaccio: reuse an existing image when available. + # Verdaccio builds always push a fresh image with a unique shadow-* tag (see below). - name: Check Docker Image id: check-image + if: ${{ !inputs.useVerdaccio || steps.lerna-changed.outputs.has_changes != 'true' }} run: | if docker manifest inspect ${{fromJSON(steps.params.outputs.result).venvImage}} > /dev/null 2>&1; then echo "Docker image exists, no need to build the image" @@ -177,6 +397,70 @@ jobs: - name: Generate npmrc run: | echo "//registry.npmjs.org/:_authToken=${{ secrets.NPM_TOKEN }}" > .npmrc + if [ "${{ inputs.useVerdaccio }}" = "true" ] && [ "${{ steps.lerna-changed.outputs.has_changes }}" = "true" ]; then + echo '@frontegg:registry=http://127.0.0.1:4873' >> .npmrc + echo '//127.0.0.1:4873/:_authToken=fake-token' >> .npmrc + fi + - name: Bump lockfile to Verdaccio packages + if: ${{ inputs.useVerdaccio && steps.lerna-changed.outputs.has_changes == 'true' }} + env: + SHADOW_VERSION: ${{ steps.publish-shadow.outputs.published_version }} + run: | + if [ -z "${SHADOW_VERSION}" ]; then + echo "::error::Missing published Verdaccio version" + exit 1 + fi + + if [ -f .yarnrc.yml ]; then + # Yarn Berry: .npmrc scoped registries are ignored; patch .yarnrc.yml directly. + node -e " + const fs = require('fs'); + const f = '.yarnrc.yml'; + let lines = fs.readFileSync(f, 'utf8').split('\n'); + // Strip existing npmScopes and unsafeHttpWhitelist blocks + let i = 0, out = []; + while (i < lines.length) { + if (/^(npmScopes|unsafeHttpWhitelist):/.test(lines[i])) { + i++; + while (i < lines.length && /^\s+/.test(lines[i])) i++; + } else { + out.push(lines[i]); + i++; + } + } + // Remove trailing blank lines then re-add config + while (out.length && out[out.length - 1].trim() === '') out.pop(); + out.push(''); + out.push('npmScopes:'); + out.push(' frontegg:'); + out.push(' npmRegistryServer: \"http://127.0.0.1:4873\"'); + out.push(' npmAuthToken: \"fake-token\"'); + out.push(''); + out.push('unsafeHttpWhitelist:'); + out.push(' - 127.0.0.1'); + out.push(''); + fs.writeFileSync(f, out.join('\n')); + " + BUMP_CMD="YARN_ENABLE_IMMUTABLE_INSTALLS=false yarn up --exact" + elif [ -f yarn.lock ]; then + BUMP_CMD="yarn upgrade" + elif [ -f package-lock.json ]; then + BUMP_CMD="npm install" + else + echo "::warning::No lockfile found; skipping lockfile bump" + exit 0 + fi + + IFS=',' read -ra PKGS <<< "${{ steps.lerna-changed.outputs.names }}" + for pkg in "${PKGS[@]}"; do + if grep -rqF "\"${pkg}\":" --include="package.json" \ + --exclude-dir=node_modules --exclude-dir=_shadow-libs --exclude-dir=.yarn .; then + echo "Bumping ${pkg} to ${SHADOW_VERSION}" + eval "${BUMP_CMD} \"${pkg}@${SHADOW_VERSION}\"" + else + echo "Skipping ${pkg}; not a direct workspace dependency" + fi + done - name: Build and push uses: docker/build-push-action@v6 if: ${{ steps.check-image.outputs.image-exist == 'false' }} @@ -187,6 +471,25 @@ jobs: build-args: | COMMIT_HASH=${{fromJSON(steps.params.outputs.result).shortSha}} NPM_TOKEN=${{ secrets.NPM_TOKEN }} + - name: Build and push (Verdaccio) + uses: docker/build-push-action@v6 + if: ${{ inputs.useVerdaccio && steps.lerna-changed.outputs.has_changes == 'true' }} + with: + context: . + push: true + network: host + tags: | + frontegg/${{fromJSON(steps.params.outputs.result).repository}}:shadow-${{github.run_id}}-${{github.run_attempt}}-${{fromJSON(steps.params.outputs.result).shortSha}} + ${{fromJSON(steps.params.outputs.result).venvImage}} + build-args: | + COMMIT_HASH=${{fromJSON(steps.params.outputs.result).shortSha}} + NPM_TOKEN=${{ secrets.NPM_TOKEN }} + - name: Push shadow tag (Verdaccio, no library changes) + if: ${{ inputs.useVerdaccio && steps.lerna-changed.outputs.has_changes != 'true' }} + run: | + docker buildx imagetools create \ + --tag frontegg/${{fromJSON(steps.params.outputs.result).repository}}:shadow-${{github.run_id}}-${{github.run_attempt}}-${{fromJSON(steps.params.outputs.result).shortSha}} \ + ${{fromJSON(steps.params.outputs.result).venvImage}} check-build-docker-results: if: ${{ always() }}