bitcoin-dev-tools · l0rinc · Nov 5, 2024 · Nov 8, 2024 · Nov 8, 2024 · Nov 8, 2024
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
diff --git a/.github/workflows/publish-results.yml b/.github/workflows/publish-results.yml
@@ -0,0 +1,256 @@
+name: Publish Results
+on:
+  workflow_run:
+    workflows: ["CI"]
+    types: [completed]
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    if: ${{ github.event.workflow_run.conclusion == 'success' }}
+    permissions:
+      actions: read
+      contents: write
+      checks: read
+    env:
+      NETWORKS: "mainnet,signet"
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: gh-pages
+      - name: Download artifacts
+        uses: actions/download-artifact@v4
+        with:
+          github-token: ${{ secrets.GH_PAT }}
+          run-id: ${{ github.event.workflow_run.id }}
+      - name: Extract artifacts
+        run: |
+          for network in ${NETWORKS//,/ }; do
+            if [ -d "result-${network}" ]; then
+              mkdir -p "${network}-results"
+              mv "result-${network}/results.json" "${network}-results/"
+            fi
+
+            if [ -d "flamegraph-${network}" ]; then
+              mkdir -p "${network}-flamegraph"
+              mv "flamegraph-${network}"/* "${network}-flamegraph/"
+            fi
+
+            if [ -d "run-metadata-${network}" ]; then
+              mkdir -p "${network}-metadata"
+              mv "run-metadata-${network}"/* "${network}-metadata/"
+            fi
+          done
+      - name: Organize results
+        id: organize
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const fs = require('fs');
+            const networks = process.env.NETWORKS.split(',');
+            let prNumber = 'main';
+            let runId;
+
+            // First, extract metadata and get PR number
+            for (const network of networks) {
+              if (fs.existsSync(`${network}-metadata/github.json`)) {
+                const metadata = JSON.parse(fs.readFileSync(`${network}-metadata/github.json`, 'utf8'));
+                prNumber = metadata.event.pull_request?.number || prNumber;
+                runId = metadata.run_id;
+              }
+            }
+
+            if (!runId) {
+              console.error('No valid metadata found for any network');
+              process.exit(1);
+            }
+
+            // Create directory structure
+            const resultDir = `results/pr-${prNumber}/${runId}`;
+            fs.mkdirSync(resultDir, { recursive: true });
+
+            // Now copy metadata files
+            for (const network of networks) {
+              if (fs.existsSync(`${network}-metadata/github.json`)) {
+                const metadataDir = `${resultDir}/${network}-metadata`;
+                fs.mkdirSync(metadataDir, { recursive: true });
+                fs.copyFileSync(`${network}-metadata/github.json`, `${metadataDir}/github.json`);
+              }
+            }
+
+            // Process each network's results
+            const combinedResults = {
+              results: []
+            };
+
+            for (const network of networks) {
+              if (fs.existsSync(`${network}-results`)) {
+                const networkResults = JSON.parse(fs.readFileSync(`${network}-results/results.json`, 'utf8'));
+
+                // Add network name to each result
+                networkResults.results.forEach(result => {
+                  result.network = network;
+                  combinedResults.results.push(result);
+                });
+
+                // Move flamegraphs
+                if (fs.existsSync(`${network}-flamegraph`)) {
+                  fs.readdirSync(`${network}-flamegraph`).forEach(file => {
+                    const sourceFile = `${network}-flamegraph/${file}`;
+                    const targetFile = `${resultDir}/${network}-${file}`;
+                    fs.copyFileSync(sourceFile, targetFile);
+                  });
+                }
+              }
+            }
+
+            // Write combined results
+            fs.writeFileSync(`${resultDir}/results.json`, JSON.stringify(combinedResults, null, 2));
+
+            // Create index.html for this run
+            const indexHtml = `<!DOCTYPE html>
+            <html>
+              <head>
+                <title>Benchmark Results</title>
+                <link href="https://cdn.jsdelivr.net/npm/tailwindcss@2.2.19/dist/tailwind.min.css" rel="stylesheet">
+              </head>
+              <body class="bg-gray-100 p-8">
+                <div class="max-w-4xl mx-auto">
+                  <h1 class="text-3xl font-bold mb-8">Benchmark Results</h1>
+                  <div class="bg-white rounded-lg shadow p-6 mb-8">
+                    <h2 class="text-xl font-semibold mb-4">PR #${prNumber} - Run ${runId}</h2>
+                    ${networks.map(network => `
+                      <div class="mb-8">
+                        <h3 class="text-lg font-semibold mb-4 capitalize">${network} Results</h3>
+                        <div class="overflow-x-auto">
+                          ${combinedResults.results
+                            .filter(result => result.network === network)
+                            .map(result => {
+                              const commitShortId = result.parameters.commit.slice(0, 8);
+                              const flameGraphFile = `${network}-${result.parameters.commit}-flamegraph.html`;
+                              const flameGraphPath = `${resultDir}/${flameGraphFile}`;
+
+                              return `
+                                <table class="min-w-full table-auto mb-4">
+                                  <thead>
+                                    <tr class="bg-gray-50">
+                                      <th class="px-4 py-2">Branch</th>
+                                      <th class="px-4 py-2">Command</th>
+                                      <th class="px-4 py-2">Mean (s)</th>
+                                      <th class="px-4 py-2">Std Dev</th>
+                                      <th class="px-4 py-2">User (s)</th>
+                                      <th class="px-4 py-2">System (s)</th>
+                                    </tr>
+                                  </thead>
+                                  <tbody>
+                                    <tr class="border-t">
+                                      <td class="px-4 py-2 font-mono text-sm">
+                                        <a href="https://github.com/bitcoin-dev-tools/benchcoin/commit/${commitShortId}">${commitShortId}</a>
+                                      </td>
+                                      <td class="px-4 py-2 font-mono text-sm">${result.command}</td>
+                                      <td class="px-4 py-2 text-right">${result.mean.toFixed(3)}</td>
+                                      <td class="px-4 py-2 text-right">${result.stddev?.toFixed(3) || 'N/A'}</td>
+                                      <td class="px-4 py-2 text-right">${result.user.toFixed(3)}</td>
+                                      <td class="px-4 py-2 text-right">${result.system.toFixed(3)}</td>
+                                    </tr>
+                                  </tbody>
+                                </table>
+
+                                ${fs.existsSync(flameGraphPath) ? `
+                                  <iframe src="${flameGraphFile}" width="100%" height="600px" frameborder="0" class="mb-4"></iframe>
+                                ` : ''}
+                              `;
+                            }).join('')}
+                        </div>
+                      </div>
+                    `).join('')}
+                  </div>
+                </div>
+              </body>
+            </html>`;
+
+            fs.writeFileSync(`${resultDir}/index.html`, indexHtml);
+
+            // Update main index.html
+            const prs = fs.readdirSync('results')
+              .filter(dir => dir.startsWith('pr-'))
+              .map(dir => ({
+                pr: dir.replace('pr-', ''),
+                runs: fs.readdirSync(`results/${dir}`)
+              }));
+
+            const mainIndexHtml = `<!DOCTYPE html>
+            <html>
+              <head>
+                <title>Bitcoin Benchmark Results</title>
+                <link href="https://cdn.jsdelivr.net/npm/tailwindcss@2.2.19/dist/tailwind.min.css" rel="stylesheet">
+              </head>
+              <body class="bg-gray-100 p-8">
+                <div class="max-w-4xl mx-auto">
+                  <h1 class="text-3xl font-bold mb-8">Bitcoin Benchmark Results</h1>
+                  <div class="bg-white rounded-lg shadow p-6">
+                    <h2 class="text-xl font-semibold mb-4">Available Results</h2>
+                    <ul class="space-y-2">
+                      ${prs.map(({pr, runs}) => `
+                        <li class="font-semibold">PR #${pr}
+                          <ul class="ml-8 space-y-1">
+                            ${runs.map(run => `
+                              <li><a href="results/pr-${pr}/${run}/index.html" class="text-blue-600 hover:underline">Run ${run}</a></li>
+                            `).join('')}
+                          </ul>
+                        </li>
+                      `).join('')}
+                    </ul>
+                  </div>
+                </div>
+              </body>
+            </html>`;
+
+            fs.writeFileSync('index.html', mainIndexHtml);
+
+            // Return the URL for the PR comment
+            const resultUrl = `https://${context.repo.owner}.github.io/${context.repo.name}/results/pr-${prNumber}/${runId}/index.html`;
+            core.setOutput('result-url', resultUrl);
+            return resultUrl;
+      - name: Upload Pages artifact
+        uses: actions/upload-pages-artifact@v3
+        with:
+          path: results
+      - name: Commit and push to gh-pages
+        run: |
+          git config --global user.name 'github-actions[bot]'
+          git config --global user.email 'github-actions[bot]@users.noreply.github.com'
+          git add results/
+          git add index.html
+          git commit -m "Update benchmark results from run ${{ github.event.workflow_run.id }}"
+          git push origin gh-pages
+  comment-pr:
+    needs: build
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
+      actions: read
+    steps:
+      - name: Download metadata artifact
+        uses: actions/download-artifact@v4
+        with:
+          pattern: run-metadata-*
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          run-id: ${{ github.event.workflow_run.id }}
+          path: metadata
+      - name: Parse Pull Request Number
+        id: parse-pr
+        run: |
+          # Find the first github.json file in any of the metadata subdirectories
+          metadata_file=$(find metadata -name github.json | head -n1)
+          if [ -n "$metadata_file" ]; then
+            pr_number=$(jq -r '.event.pull_request.number' "$metadata_file")
+            echo "PR_NUMBER=$pr_number" >> "$GITHUB_ENV"
+          fi
+      - name: Comment on PR
+        if: ${{ env.PR_NUMBER }}
+        uses: thollander/actions-comment-pull-request@v3.0.1
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          pr-number: ${{ env.PR_NUMBER }}
+          message: |
+            📊 Benchmark results for this run (${{ github.event.workflow_run.id }}) will be available at: https://${{ github.repository_owner }}.github.io/${{ github.event.repository.name }}/results/pr-${{ env.PR_NUMBER }}/${{ github.event.workflow_run.id }}/index.html after the github pages "build and deployment" action has completed.
diff --git a/bench-ci/run-assumeutxo-bench.sh b/bench-ci/run-assumeutxo-bench.sh
@@ -0,0 +1,141 @@
+#!/usr/bin/env bash
+
+set -euxo pipefail
+
+# Helper function to check and clean datadir
+clean_datadir() {
+  set -euxo pipefail
+
+  local TMP_DATADIR="$1"
+
+  # Create the directory if it doesn't exist
+  mkdir -p "${TMP_DATADIR}"
+
+  # If we're in CI, clean without confirmation
+  if [ -n "${CI:-}" ]; then
+    rm -Rf "${TMP_DATADIR:?}"/*
+  else
+    read -rp "Are you sure you want to delete everything in ${TMP_DATADIR}? [y/N] " response
+    if [[ "$response" =~ ^[Yy]$ ]]; then
+      rm -Rf "${TMP_DATADIR:?}"/*
+    else
+      echo "Aborting..."
+      exit 1
+    fi
+  fi
+}
+
+# Helper function to clear logs
+clean_logs() {
+  set -euxo pipefail
+
+  local TMP_DATADIR="$1"
+  local logfile="${TMP_DATADIR}/debug.log"
+
+  echo "Checking for ${logfile}"
+  if [ -e "{$logfile}" ]; then
+    echo "Removing ${logfile}"
+    rm "${logfile}"
+  fi
+}
+
+# Execute CMD before each set of timing runs.
+setup_assumeutxo_snapshot_run() {
+  set -euxo pipefail
+
+  local commit="$1"
+  local TMP_DATADIR="$2"
+
+  git checkout "${commit}"
+  ccache -z
+  ccache -s
+  cmake -B build \
+      -DBUILD_BENCH=OFF \
+      -DBUILD_TESTS=OFF \
+      -DCMAKE_BUILD_TYPE=RelWithDebInfo \
+      -DCMAKE_CXX_FLAGS="-fno-omit-frame-pointer" \
+      -DCMAKE_C_COMPILER_LAUNCHER=ccache \
+      -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
+  cmake --build build -j "$(nproc)"
+  ccache -s
+  clean_datadir "${TMP_DATADIR}"
+}
+
+# Execute CMD before each timing run.
+prepare_assumeutxo_snapshot_run() {
+  set -euxo pipefail
+
+  local TMP_DATADIR="$1"
+  local UTXO_PATH="$2"
+  local CONNECT_ADDRESS="$3"
+  local chain="$4"
+
+  # Run the actual preparation steps
+  clean_datadir "${TMP_DATADIR}"
+  build/src/bitcoind -datadir="${TMP_DATADIR}" -connect="${CONNECT_ADDRESS}" -daemon=0 -chain="${chain}" -stopatheight=1
+  build/src/bitcoind -datadir="${TMP_DATADIR}" -connect="${CONNECT_ADDRESS}" -daemon=0 -chain="${chain}" -dbcache=16000 -pausebackgroundsync=1 -loadutxosnapshot="${UTXO_PATH}" || true
+  clean_logs "${TMP_DATADIR}"
+}
+
+# Executed after each timing run
+conclude_assumeutxo_snapshot_run() {
+  set -euxo pipefail
+
+  local commit="$1"
+
+  if [ -e flamegraph.html ]; then
+    mv flamegraph.html "${commit}"-flamegraph.html
+  fi
+}
+
+# Execute CMD after the completion of all benchmarking runs for each individual
+# command to be benchmarked.
+cleanup_assumeutxo_snapshot_run() {
+  set -euxo pipefail
+
+  local TMP_DATADIR="$1"
+
+  # Clean up the datadir
+  clean_datadir "${TMP_DATADIR}"
+}
+
+run_benchmark() {
+  local base_commit="$1"
+  local head_commit="$2"
+  local TMP_DATADIR="$3"
+  local UTXO_PATH="$4"
+  local results_file="$5"
+  local chain="$6"
+  local stop_at_height="$7"
+  local connect_address="$8"
+
+  # Export functions so they can be used by hyperfine
+  export -f setup_assumeutxo_snapshot_run
+  export -f prepare_assumeutxo_snapshot_run
+  export -f conclude_assumeutxo_snapshot_run
+  export -f cleanup_assumeutxo_snapshot_run
+  export -f clean_datadir
+  export -f clean_logs
+
+  # Run hyperfine
+  hyperfine \
+    --setup "setup_assumeutxo_snapshot_run {commit} ${TMP_DATADIR}" \
+    --prepare "prepare_assumeutxo_snapshot_run ${TMP_DATADIR} ${UTXO_PATH} ${connect_address} ${chain}" \
+    --conclude "conclude_assumeutxo_snapshot_run {commit}" \
+    --cleanup "cleanup_assumeutxo_snapshot_run ${TMP_DATADIR}" \
+    --runs 1 \
+    --show-output \
+    --export-json "${results_file}" \
+    --command-name "base (${base_commit})" \
+    --command-name "head (${head_commit})" \
+    "taskset -c 1 perf script flamegraph taskset -c 2-15 build/src/bitcoind -datadir=${TMP_DATADIR} -connect=${connect_address} -daemon=0 -chain=${chain} -stopatheight=${stop_at_height}" \
+    -L commit "${base_commit},${head_commit}"
+}
+
+# Main execution
+if [ "$#" -ne 8 ]; then
+  echo "Usage: $0 base_commit head_commit TMP_DATADIR UTXO_PATH results_dir chain stop_at_height connect_address"
+  exit 1
+fi
+
+run_benchmark "$1" "$2" "$3" "$4" "$5" "$6" "$7" "$8"