[WIP] perf: optimize boot time with lazy imports and combined DB queries

.envrc

@@ -0,0 +1 @@
+use flake

.github/workflows/build.yml

@@ -0,0 +1,150 @@
+name: Build PiFinder NixOS
+
+on:
+  push:
+    branches: [main, nixos]
+  pull_request:
+    types: [labeled, synchronize, opened]
+  workflow_dispatch:
+
+concurrency:
+  group: build-${{ github.head_ref || github.ref_name }}
+  cancel-in-progress: true
+
+jobs:
+  # Try Pi5 native build first (fast)
+  build-native:
+    if: |
+      github.event_name == 'push' ||
+      github.event_name == 'workflow_dispatch' ||
+      contains(github.event.pull_request.labels.*.name, 'preview') ||
+      contains(github.event.pull_request.labels.*.name, 'testable')
+    runs-on: [self-hosted, aarch64]
+    timeout-minutes: 30
+    outputs:
+      success: ${{ steps.build.outcome == 'success' }}
+      store_path: ${{ steps.push.outputs.store_path }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: cachix/cachix-action@v15
+        with:
+          name: pifinder
+          authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
+
+      - name: Build NixOS system closure
+        id: build
+        run: |
+          nix build .#nixosConfigurations.pifinder.config.system.build.toplevel \
+            -L --no-link
+
+      - name: Push to Cachix
+        id: push
+        run: |
+          STORE_PATH=$(nix build .#nixosConfigurations.pifinder.config.system.build.toplevel \
+            --json | jq -r '.[].outputs.out')
+          echo "$STORE_PATH" | cachix push pifinder
+          echo "store_path=$STORE_PATH" >> "$GITHUB_OUTPUT"
+
+  # Wait up to 15 min for native builder, then decide on fallback
+  native-wait:
+    if: |
+      github.event_name == 'push' ||
+      github.event_name == 'workflow_dispatch' ||
+      contains(github.event.pull_request.labels.*.name, 'preview') ||
+      contains(github.event.pull_request.labels.*.name, 'testable')
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    outputs:
+      need_emulated: ${{ steps.wait.outputs.need_emulated }}
+    steps:
+      - name: Wait for native build
+        id: wait
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          for i in $(seq 1 30); do
+            sleep 30
+            RESULT=$(gh api "repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/jobs" \
+              --jq '.jobs[] | select(.name == "build-native") | .conclusion // "pending"' 2>/dev/null || echo "pending")
+            echo "Check $i/30: build-native=$RESULT"
+            if [ "$RESULT" = "success" ]; then
+              echo "need_emulated=false" >> "$GITHUB_OUTPUT"
+              exit 0
+            elif [ "$RESULT" = "failure" ] || [ "$RESULT" = "cancelled" ]; then
+              echo "need_emulated=true" >> "$GITHUB_OUTPUT"
+              exit 0
+            fi
+          done
+          echo "Native build not done after 15 min, falling back to emulated"
+          echo "need_emulated=true" >> "$GITHUB_OUTPUT"
+
+  # Fallback to QEMU emulation if Pi5 unavailable or slow
+  build-emulated:
+    needs: native-wait
+    if: needs.native-wait.outputs.need_emulated == 'true'
+    runs-on: ubuntu-latest
+    timeout-minutes: 360
+    outputs:
+      store_path: ${{ steps.push.outputs.store_path }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: cachix/install-nix-action@v27
+        with:
+          extra_nix_config: |
+            extra-platforms = aarch64-linux
+            extra-system-features = big-parallel
+
+      - uses: cachix/cachix-action@v15
+        with:
+          name: pifinder
+          authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
+
+      - name: Register QEMU binfmt for aarch64
+        run: sudo apt-get update && sudo apt-get install -y qemu-user-static
+
+      - name: Build NixOS system closure
+        run: |
+          nix build .#nixosConfigurations.pifinder.config.system.build.toplevel \
+            --system aarch64-linux \
+            -L --no-link
+
+      - name: Push to Cachix
+        id: push
+        run: |
+          STORE_PATH=$(nix build .#nixosConfigurations.pifinder.config.system.build.toplevel \
+            --system aarch64-linux \
+            --json | jq -r '.[].outputs.out')
+          echo "$STORE_PATH" | cachix push pifinder
+          echo "store_path=$STORE_PATH" >> "$GITHUB_OUTPUT"
+
+  # Commit pifinder-build.json with store path to the same branch
+  stamp-build:
+    needs: [build-native, build-emulated]
+    if: always() && (needs.build-native.result == 'success' || needs.build-emulated.result == 'success')
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.head_ref || github.ref_name }}
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Write pifinder-build.json
+        run: |
+          STORE_PATH="${{ needs.build-native.outputs.store_path || needs.build-emulated.outputs.store_path }}"
+          BRANCH="${{ github.head_ref || github.ref_name }}"
+          SHORT_SHA=$(git rev-parse --short HEAD)
+          VERSION="${BRANCH}-${SHORT_SHA}"
+          jq -n --arg sp "$STORE_PATH" --arg v "$VERSION" \
+            '{store_path: $sp, version: $v}' > pifinder-build.json
+
+      - name: Commit pifinder-build.json
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          git add pifinder-build.json
+          git diff --staged --quiet || git commit -m "chore: stamp build [skip ci]"
+          git push

.github/workflows/lint.yml

@@ -0,0 +1,56 @@
+name: Lint & Test
+on: [push, pull_request]
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - uses: DeterminateSystems/nix-installer-action@main
+        with:
+          determinate: false
+      - uses: DeterminateSystems/magic-nix-cache-action@main
+      - name: Lint
+        run: nix develop --command bash -c "cd python && ruff check"
+      - name: Format check
+        run: nix develop --command bash -c "cd python && ruff format --check"
+      - name: Check for removed config keys
+        if: github.event_name == 'pull_request'
+        run: |
+          BASE_SHA="${{ github.event.pull_request.base.sha }}"
+          git show "$BASE_SHA:default_config.json" 2>/dev/null \
+            | python3 -c "import json,sys; print('\n'.join(sorted(json.load(sys.stdin).keys())))" \
+            > /tmp/base_keys.txt || exit 0
+          python3 -c "import json,sys; print('\n'.join(sorted(json.load(sys.stdin).keys())))" \
+            < default_config.json > /tmp/head_keys.txt
+          REMOVED=$(comm -23 /tmp/base_keys.txt /tmp/head_keys.txt)
+          if [ -n "$REMOVED" ]; then
+            while IFS= read -r key; do
+              echo "::warning file=default_config.json::Config key '$key' was removed — this may break user preferences across release switches"
+            done <<< "$REMOVED"
+          fi
+
+  type-check:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: DeterminateSystems/nix-installer-action@main
+        with:
+          determinate: false
+      - uses: DeterminateSystems/magic-nix-cache-action@main
+      - name: Type check
+        run: nix develop --command bash -c "cd python && mypy --install-types --non-interactive ."
+
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: DeterminateSystems/nix-installer-action@main
+        with:
+          determinate: false
+      - uses: DeterminateSystems/magic-nix-cache-action@main
+      - name: Smoke tests
+        run: nix develop --command bash -c "cd python && pytest -m smoke"
+      - name: Unit tests
+        run: nix develop --command bash -c "cd python && pytest -m unit"

.github/workflows/release.yml

@@ -0,0 +1,132 @@
+name: Release
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'Version (e.g., 2.5.0)'
+        required: true
+      notes:
+        description: 'Release notes'
+        required: true
+      type:
+        description: 'Release type'
+        type: choice
+        options:
+          - stable
+          - beta
+        default: stable
+      source_branch:
+        description: 'Source branch (default: main, use release/X.Y for hotfixes)'
+        required: false
+        default: 'main'
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.source_branch }}
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - uses: cachix/install-nix-action@v27
+        with:
+          extra_nix_config: |
+            extra-platforms = aarch64-linux
+            extra-system-features = big-parallel
+
+      - uses: cachix/cachix-action@v15
+        with:
+          name: pifinder
+          authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
+
+      - name: Register QEMU binfmt for aarch64
+        run: sudo apt-get update && sudo apt-get install -y qemu-user-static
+
+      - name: Build and push closure to Cachix
+        id: push
+        run: |
+          STORE_PATH=$(nix build .#nixosConfigurations.pifinder.config.system.build.toplevel \
+            --system aarch64-linux -L --json | jq -r '.[].outputs.out')
+          echo "$STORE_PATH" | cachix push pifinder
+          echo "store_path=$STORE_PATH" >> "$GITHUB_OUTPUT"
+
+      - name: Stamp pifinder-build.json and create tag
+        run: |
+          TAG="v${{ inputs.version }}"
+          [[ "${{ inputs.type }}" == "beta" ]] && TAG="${TAG}-beta"
+
+          STORE_PATH="${{ steps.push.outputs.store_path }}"
+          VERSION="${{ inputs.version }}"
+          jq -n --arg sp "$STORE_PATH" --arg v "$VERSION" \
+            '{store_path: $sp, version: $v}' > pifinder-build.json
+
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          git add pifinder-build.json
+          git diff --staged --quiet || git commit -m "release: stamp build for $TAG [skip ci]"
+          git push origin "${{ inputs.source_branch }}"
+
+          git tag "$TAG"
+          git push origin "$TAG"
+          echo "TAG=$TAG" >> $GITHUB_ENV
+
+      - name: Build SD image
+        run: |
+          nix build .#images.pifinder \
+            --system aarch64-linux \
+            -L -o result-sd
+          mkdir -p release
+          for f in result-sd/sd-image/*.img.zst; do
+            cp "$f" "release/pifinder-${TAG}.img.zst"
+          done
+
+      - name: Build migration tarball from SD image
+        run: |
+          IMAGE_FILE=$(find result-sd/sd-image -type f \( -name "*.img" -o -name "*.img.zst" \) | head -1)
+
+          rm -f /tmp/pifinder-release.img
+          if [[ "${IMAGE_FILE}" == *.zst ]]; then
+            zstd -d "${IMAGE_FILE}" -o /tmp/pifinder-release.img
+          else
+            cp "${IMAGE_FILE}" /tmp/pifinder-release.img
+          fi
+
+          LOOP=$(sudo losetup --find --show --partscan /tmp/pifinder-release.img)
+          sudo mkdir -p /mnt/boot /mnt/root
+          sudo mount "${LOOP}p1" /mnt/boot
+          sudo mount "${LOOP}p2" /mnt/root
+
+          mkdir -p /tmp/tarball-staging
+          sudo cp -a /mnt/boot /tmp/tarball-staging/boot
+          sudo cp -a /mnt/root /tmp/tarball-staging/rootfs
+          sudo rm -rf /tmp/tarball-staging/rootfs/home/pifinder/PiFinder_data/catalog_images
+
+          sudo umount /mnt/boot /mnt/root
+          sudo losetup -d "${LOOP}"
+          rm -f /tmp/pifinder-release.img
+
+          sudo tar -C /tmp/tarball-staging -cf - \
+            --exclude='*/lost+found' \
+            boot rootfs | zstd -T0 -19 -o "release/pifinder-migration-${TAG}.tar.zst"
+          sudo rm -rf /tmp/tarball-staging
+
+      - name: Upload build artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: pifinder-release-${{ env.TAG }}
+          path: release/pifinder-*.zst
+          retention-days: 90
+
+      - name: Create GitHub Release
+        uses: softprops/action-gh-release@v1
+        with:
+          tag_name: ${{ env.TAG }}
+          name: PiFinder ${{ env.TAG }}
+          body: ${{ inputs.notes }}
+          prerelease: ${{ inputs.type == 'beta' }}
+          files: |
+            release/pifinder-*.zst

.pre-commit-config.yaml

@@ -1,15 +1,27 @@
-# See https://pre-commit.com for more information
-# See https://pre-commit.com/hooks.html for more hooks
 repos:
-  - repo: https://github.com/saltstack/mirrors-nox
-    rev: 'v2022.11.21'  # Use the sha / tag you want to point at
+  - repo: local
     hooks:
-      - id: nox
-        files: ^.*\.py$
-        args:
-          - -f
-          - python/noxfile.py
-          - -s
-          - type_hints
-          - smoke_tests
-          - --
+      - id: ruff-lint
+        name: ruff lint
+        entry: bash -c 'cd python && ruff check'
+        language: system
+        files: ^python/.*\.py$
+        pass_filenames: false
+      - id: ruff-format
+        name: ruff format check
+        entry: bash -c 'cd python && ruff format --check'
+        language: system
+        files: ^python/.*\.py$
+        pass_filenames: false
+      - id: mypy
+        name: mypy type check
+        entry: bash -c 'cd python && mypy .'
+        language: system
+        files: ^python/.*\.py$
+        pass_filenames: false
+      - id: smoke-tests
+        name: smoke tests
+        entry: bash -c 'cd python && pytest -m smoke'
+        language: system
+        files: ^python/.*\.py$
+        pass_filenames: false