From 8c65e0afff23e455794ae3c548388660988aaaa2 Mon Sep 17 00:00:00 2001 From: rosaliedewinther Date: Thu, 13 Mar 2025 21:08:01 +0100 Subject: [PATCH 01/10] towards improved graphs --- scripts/Untitled.ipynb | 412 +++++++++++++++++++++++++++++++ scripts/compare_deep_analysis.py | 15 +- 2 files changed, 421 insertions(+), 6 deletions(-) create mode 100644 scripts/Untitled.ipynb diff --git a/scripts/Untitled.ipynb b/scripts/Untitled.ipynb new file mode 100644 index 0000000..e7eea2a --- /dev/null +++ b/scripts/Untitled.ipynb @@ -0,0 +1,412 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "aadf8bdc-2b8a-4737-992f-0ebf7e66cd8f", + "metadata": {}, + "outputs": [], + "source": [ + "%%cmd\n", + "python -m pip install flatten_json" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bbdc923a-44fd-4250-990b-80771446bbf0", + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import json\n", + "from collections import defaultdict\n", + "import numpy as np\n", + "import argparse\n", + "import sys\n", + "from flatten_json import flatten\n", + "import matplotlib.pyplot as plt" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "099d7d0d-febc-4562-801a-767c4b028f56", + "metadata": {}, + "outputs": [], + "source": [ + "def aggregate_loops_passes(json):\n", + " results_per_frame = []\n", + " num_loops = len(json)\n", + " for loop_results in json:\n", + " for frame_index, frame_results in enumerate(loop_results[\"per_frame_results\"]):\n", + " #pprint.pprint(frame_results)\n", + " if frame_index >= len(results_per_frame):\n", + " results_per_frame.append(defaultdict(int))\n", + " for command_buffer_timings in frame_results[\n", + " \"command_buffer_timings\"\n", + " ].values():\n", + " for scope_name, scope_timings in command_buffer_timings[\n", + " \"scope_timings\"\n", + " ].items():\n", + " for scope_timing in scope_timings:\n", + " results_per_frame[frame_index][scope_name] += (\n", + " scope_timing[\"end\"] - scope_timing[\"start\"]\n", + " ) / num_loops / 1_000_000 # in ms\n", + " for metric_name, metric in frame_results[\"metrics\"].items():\n", + " # TODO: Flatten this in rust to fan_speed_rpm\n", + " if metric_name == \"fan_speed\":\n", + " value = metric[\"Percent\"] if \"Percent\" in metric else metric[\"Rpm\"]\n", + " results_per_frame[frame_index][\"fan_speed_rpm\"] += (\n", + " value / num_loops\n", + " )\n", + " # Filter out unavailable data and the timestamp\n", + " elif metric is not None and metric_name != \"timestamp\":\n", + " results_per_frame[frame_index][metric_name] += metric / num_loops\n", + " # TODO: Aggregate CPU timings\n", + " return pd.DataFrame([flatten(x) for x in results_per_frame])\n", + "\n", + "\n", + "def metric_names():\n", + " return [\n", + " \"edge_temperature_in_c\",\n", + " \"hotspot_temperature_in_c\",\n", + " \"usage_percentage\",\n", + " \"fan_speed_rpm\",\n", + " \"clock_speed_in_mhz\",\n", + " \"vram_clock_speed_in_mhz\",\n", + " \"board_power_usage_in_w\",\n", + " \"voltage_in_mv\",\n", + " \"vram_usage_in_mb\",\n", + " ]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "939b04cb-7ffd-4963-8dac-71c0a76275cf", + "metadata": {}, + "outputs": [], + "source": [ + "files = [\n", + " #'../../a_path_tracing_workgraph.csv',\n", + " #'../../b_path_tracing_workgraph.csv',\n", + " '../../a_ray_tracing_inline.csv',\n", + " '../../b_ray_tracing_inline.csv',\n", + " '../../a_ray_tracing_pipeline.csv',\n", + " '../../b_ray_tracing_pipeline.csv',\n", + " '../../b_loop.csv'\n", + "]\n", + "\n", + "scores = {}\n", + "\n", + "for file in files:\n", + " scores[file] = pd.read_csv(file)\n", + "scores = pd.concat(scores).groupby(level=0, sort=False).mean().T.drop('Loop', errors=\"ignore\")\n", + "scores" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2fbaaf11-5b32-43ed-84d1-8894ec6de013", + "metadata": {}, + "outputs": [], + "source": [ + "scores.plot(kind=\"bar\", figsize=(20,5), colormap='Dark2', grid=True, rot=0)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a60a4c63-b884-4558-98bd-ee3d14bbc569", + "metadata": {}, + "outputs": [], + "source": [ + "files = [\n", + " '../../a_ray_tracing_inline_deep.json',\n", + " '../../b_ray_tracing_inline_deep.json',\n", + " #'../../a_ray_tracing_pipeline_deep.json',\n", + " #'../../b_ray_tracing_pipeline_deep.json',\n", + " #'../../a_path_tracing_workgraph_deep.json',\n", + " #'../../b_path_tracing_workgraph_deep.json',\n", + "]\n", + "\n", + "results = {}\n", + "\n", + "for path in files:\n", + " with open(path, \"r\") as json_file:\n", + " json_data = aggregate_loops_passes(json.load(json_file))\n", + " results[path] = json_data\n", + "result = pd.concat(results)\n", + "result" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d3e81b11-e754-44ad-9bb0-3b08abe4dddd", + "metadata": {}, + "outputs": [], + "source": [ + "result.columns.tolist()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6046a029-f421-4a9e-a9d6-9889488ff874", + "metadata": {}, + "outputs": [], + "source": [ + "relevant_metrics = [\n", + " 'apply_wind_to_vertices',\n", + " 'usage_percentage',\n", + " 'vbuffer-mesh-parts-prefix-sum'\n", + "]\n", + "metrics = result[metric_names()]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f99fd1c3-a1fb-440c-b428-a7f9da8dce8b", + "metadata": {}, + "outputs": [], + "source": [ + "filtered_metrics = metrics.stack().unstack(level=0).reset_index()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f4085d0a-b087-469b-8aeb-dce0be19c41b", + "metadata": {}, + "outputs": [], + "source": [ + "filtered_metrics.groupby('level_1').plot(x='level_0', xlabel='frame', figsize=(20,5), colormap='Dark2', grid=True, legend=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e6740136-668a-44a6-9e35-53f80d586ea5", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a27cb615-711d-47b7-b570-9fd2da3efb2f", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0d18b49a-bb26-4dac-8c8d-f50f6d70fa3d", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "06b1a356-48f5-4a26-bfc6-0d34cdeaf3a4", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0936c27b-d76a-4d2b-8acf-a7f9cfd957c9", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bc7b1605-197a-4af5-928f-a59621265541", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e7d880f6-fbe6-4c82-9669-d55371add589", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "86ed4d62-2fbd-463b-bc98-8d25620dce33", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8f2ed3de-a804-48f8-bbd9-a0e9aa1c1461", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "33e1c5a6-6494-4915-9138-7e005b13e784", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5bbcc90f-4d88-44a0-9941-aa11bc400ebb", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "45778568-9d34-4ad8-8122-546f171e0e9a", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b5d46180-fa10-43ca-87ea-085321bd59ec", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3e563c23-2991-4b6b-9c60-49f497e02474", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8090f30c-6053-46b9-b110-ee0ca7748b1e", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a7fec099-b121-43f2-8ef6-ee7ef89ba690", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6d715819-9d82-42d0-9ee1-0766661e7e63", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e2c91a7f-5948-4446-b995-6f7b301f618d", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5e8a505b-4a73-4e6a-bc28-4e0f3adfbbdd", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "28bd4976-bb47-4287-858a-17449717f2d2", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ffa75d91-f087-47e4-abee-a9335ba1f81a", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7f84c673-e6d9-4df0-b911-f3d29119c8f7", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "28d8ec51-da05-49ca-bb9e-e6644d2fc3a9", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "178eb511-0039-4a72-97f2-02c51d267036", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e1f27031-080e-49bb-b1a4-91b3bd9fa0da", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.4" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/scripts/compare_deep_analysis.py b/scripts/compare_deep_analysis.py index 91df63e..eeeb97b 100644 --- a/scripts/compare_deep_analysis.py +++ b/scripts/compare_deep_analysis.py @@ -4,6 +4,7 @@ import numpy as np import argparse import sys +import pprint from flatten_json import flatten @@ -17,6 +18,7 @@ def aggregate_loops_passes(json): num_loops = len(json) for loop_results in json: for frame_index, frame_results in enumerate(loop_results["per_frame_results"]): + #pprint.pprint(frame_results) if frame_index >= len(results_per_frame): results_per_frame.append(defaultdict(int)) for command_buffer_timings in frame_results[ @@ -32,8 +34,9 @@ def aggregate_loops_passes(json): for metric_name, metric in frame_results["metrics"].items(): # TODO: Flatten this in rust to fan_speed_rpm if metric_name == "fan_speed": + value = metric["Percent"] if "Percent" in metric else metric["Rpm"] results_per_frame[frame_index]["fan_speed_rpm"] += ( - metric["Rpm"] / num_loops + value / num_loops ) # Filter out unavailable data and the timestamp elif metric is not None and metric_name != "timestamp": @@ -69,10 +72,10 @@ def output_top_passes(input_1, input_2): combined_data.index.names = ["Pass Name"] combined_data["pct_diff"] = ( combined_data["Input 1"] / combined_data["Input 2"] - ).abs().drop(metric_names()) * 100 + ).abs().drop(metric_names(), errors="ignore") * 100 combined_data = ( combined_data.sort_values(by="pct_diff", ascending=False) - .iloc[:20] + #.iloc[:20] .round() .dropna() .astype(int) @@ -97,10 +100,10 @@ def output_top_stdev(input_1, input_2): # depending on the set language combined_data["pct_diff"] = ( combined_data["Input 1"] / combined_data["Input 2"] - ).abs().drop(metric_names()) * 100 + ).abs().drop(metric_names(), errors="ignore") * 100 combined_data = ( combined_data.sort_values(by="pct_diff", ascending=False) - .iloc[:20] + #.iloc[:20] .round() .dropna() .astype(int) @@ -140,7 +143,7 @@ def main(): parser.print_usage() exit(2) - if args.pass_mean_comparison is None and args.pass_stdev_comparison is None: + if not args.pass_mean_comparison and not args.pass_stdev_comparison: print("Error: No analysis option specified, specify at least one\n") parser.print_usage() exit(1) From 1c70004342458a9463e8e6272c475a05db40b179 Mon Sep 17 00:00:00 2001 From: rosaliedewinther Date: Mon, 17 Mar 2025 15:16:59 +0100 Subject: [PATCH 02/10] fix-example --- ...tled.ipynb => deep_analysis_example.ipynb} | 111 ++++++++++++------ 1 file changed, 73 insertions(+), 38 deletions(-) rename scripts/{Untitled.ipynb => deep_analysis_example.ipynb} (73%) diff --git a/scripts/Untitled.ipynb b/scripts/deep_analysis_example.ipynb similarity index 73% rename from scripts/Untitled.ipynb rename to scripts/deep_analysis_example.ipynb index e7eea2a..45dbffd 100644 --- a/scripts/Untitled.ipynb +++ b/scripts/deep_analysis_example.ipynb @@ -25,7 +25,8 @@ "import argparse\n", "import sys\n", "from flatten_json import flatten\n", - "import matplotlib.pyplot as plt" + "import matplotlib.pyplot as plt\n", + "import math" ] }, { @@ -43,6 +44,7 @@ " #pprint.pprint(frame_results)\n", " if frame_index >= len(results_per_frame):\n", " results_per_frame.append(defaultdict(int))\n", + " results_per_frame[frame_index]['sequence_time_ns'] = frame_results['sequence_time_ns']\n", " for command_buffer_timings in frame_results[\n", " \"command_buffer_timings\"\n", " ].values():\n", @@ -89,19 +91,20 @@ "outputs": [], "source": [ "files = [\n", - " #'../../a_path_tracing_workgraph.csv',\n", - " #'../../b_path_tracing_workgraph.csv',\n", " '../../a_ray_tracing_inline.csv',\n", - " '../../b_ray_tracing_inline.csv',\n", " '../../a_ray_tracing_pipeline.csv',\n", + " '../../b_ray_tracing_inline.csv',\n", " '../../b_ray_tracing_pipeline.csv',\n", - " '../../b_loop.csv'\n", "]\n", "\n", "scores = {}\n", "\n", "for file in files:\n", - " scores[file] = pd.read_csv(file)\n", + " # Read csv\n", + " file_scores = pd.read_csv(file)\n", + " # Remove empty results\n", + " file_scores = file_scores.loc[:, (file_scores != 0).any(axis=0)]\n", + " scores[file] = file_scores\n", "scores = pd.concat(scores).groupby(level=0, sort=False).mean().T.drop('Loop', errors=\"ignore\")\n", "scores" ] @@ -126,30 +129,20 @@ "files = [\n", " '../../a_ray_tracing_inline_deep.json',\n", " '../../b_ray_tracing_inline_deep.json',\n", - " #'../../a_ray_tracing_pipeline_deep.json',\n", - " #'../../b_ray_tracing_pipeline_deep.json',\n", - " #'../../a_path_tracing_workgraph_deep.json',\n", - " #'../../b_path_tracing_workgraph_deep.json',\n", + " '../../a_ray_tracing_pipeline_deep.json',\n", + " '../../b_ray_tracing_pipeline_deep.json',\n", "]\n", "\n", "results = {}\n", "\n", + "# Load all files into one large dataframe\n", "for path in files:\n", " with open(path, \"r\") as json_file:\n", + " # We agregate passes within each frame, so we get one number per pass per frame per input file\n", " json_data = aggregate_loops_passes(json.load(json_file))\n", " results[path] = json_data\n", - "result = pd.concat(results)\n", - "result" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d3e81b11-e754-44ad-9bb0-3b08abe4dddd", - "metadata": {}, - "outputs": [], - "source": [ - "result.columns.tolist()" + "full_dataset = pd.concat(results)\n", + "full_dataset" ] }, { @@ -160,41 +153,83 @@ "outputs": [], "source": [ "relevant_metrics = [\n", - " 'apply_wind_to_vertices',\n", - " 'usage_percentage',\n", - " 'vbuffer-mesh-parts-prefix-sum'\n", + " 'reflection-hits-shading',\n", + " 'water-compositing',\n", + " 'blur',\n", + " 'diffuse-spatial-filter',\n", + " 'spd',\n", + " 'sun-direct-lighting',\n", + " 'reflection-ray-tracing-inline',\n", + " 'trace-diffuse-nee-rays',\n", + " 'render pass',\n", + " 'shadow-ray-tracing-pipeline',\n", + " 'compositing',\n", + " 'build-gbuffers_0',\n", + " 'scale-raster',\n", + " 'Batch refit bottom level',\n", + " 'clock_speed_in_mhz',\n", + " 'board_power_usage_in_w',\n", + " 'vram_usage_in_mb',\n", + " 'edge_temperature_in_c'\n", "]\n", - "metrics = result[metric_names()]" + "# We want all relevant metrics with the sequence time, to properly plot on the x axis\n", + "relevant_metrics_with_time = relevant_metrics + [\"sequence_time_ns\"]\n", + "metrics = full_dataset[relevant_metrics_with_time]\n", + "\n", + "# Use sequence time and input name as index, each row will have a unique time and input\n", + "metrics = metrics.reset_index().set_index(['sequence_time_ns', 'level_0']).drop('level_1', axis=1)\n", + "\n", + "# Make each input file a column, and put the pass names into a specific column\n", + "metrics = metrics.stack().unstack(1).reset_index()\n", + "\n", + "# From nano seconds to seconds\n", + "metrics['sequence_time_ns'] = metrics['sequence_time_ns'] / 1_000_000_000\n", + "metrics" ] }, { "cell_type": "code", "execution_count": null, - "id": "f99fd1c3-a1fb-440c-b428-a7f9da8dce8b", + "id": "217b62b6-2500-45b5-b3f0-24449387fddd", "metadata": {}, "outputs": [], "source": [ - "filtered_metrics = metrics.stack().unstack(level=0).reset_index()" + "# Print all possible metrics\n", + "full_dataset.columns.tolist()" ] }, { "cell_type": "code", "execution_count": null, - "id": "f4085d0a-b087-469b-8aeb-dce0be19c41b", + "id": "e6740136-668a-44a6-9e35-53f80d586ea5", "metadata": {}, "outputs": [], "source": [ - "filtered_metrics.groupby('level_1').plot(x='level_0', xlabel='frame', figsize=(20,5), colormap='Dark2', grid=True, legend=True)" + "for graph_name in metrics['level_1'].unique():\n", + " # Grab the metric we want to plot\n", + " selected_metric = metrics[metrics['level_1'] == graph_name]\n", + " selected_metric = filtered_data.drop('level_1', axis=1)\n", + "\n", + " # Filter outliers out of view\n", + " max_mean = selected_metric[files].mean().mean()\n", + " max_mean = 0 if pd.isna(max_mean) else max_mean\n", + " max_std = selected_metric[files].std(axis=1).max()\n", + " max_std = selected_metric[files].max() / 3.0 if pd.isna(max_std) else max_std\n", + "\n", + " # Plot results \n", + " selected_metric.infer_objects(copy=False).interpolate(method='linear').plot(\n", + " x='sequence_time_ns', \n", + " ylabel='shader execution time in ms',\n", + " xlabel='benchmark timeline in seconds', \n", + " ylim= (max(0, max_mean - max_std * 3), max_mean + max_std * 3),\n", + " figsize=(20,10), \n", + " colormap='Dark2', \n", + " grid=True, \n", + " legend=True,\n", + " title=graph_name\n", + " )" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "e6740136-668a-44a6-9e35-53f80d586ea5", - "metadata": {}, - "outputs": [], - "source": [] - }, { "cell_type": "code", "execution_count": null, From 9dcffbf8b19c67beee98aa1cf9090cda7fdbeec9 Mon Sep 17 00:00:00 2001 From: rosaliedewinther Date: Mon, 17 Mar 2025 17:45:55 +0100 Subject: [PATCH 03/10] remove empty lines and revert file --- scripts/compare_deep_analysis.py | 15 +-- scripts/deep_analysis_example.ipynb | 192 ---------------------------- 2 files changed, 6 insertions(+), 201 deletions(-) diff --git a/scripts/compare_deep_analysis.py b/scripts/compare_deep_analysis.py index eeeb97b..91df63e 100644 --- a/scripts/compare_deep_analysis.py +++ b/scripts/compare_deep_analysis.py @@ -4,7 +4,6 @@ import numpy as np import argparse import sys -import pprint from flatten_json import flatten @@ -18,7 +17,6 @@ def aggregate_loops_passes(json): num_loops = len(json) for loop_results in json: for frame_index, frame_results in enumerate(loop_results["per_frame_results"]): - #pprint.pprint(frame_results) if frame_index >= len(results_per_frame): results_per_frame.append(defaultdict(int)) for command_buffer_timings in frame_results[ @@ -34,9 +32,8 @@ def aggregate_loops_passes(json): for metric_name, metric in frame_results["metrics"].items(): # TODO: Flatten this in rust to fan_speed_rpm if metric_name == "fan_speed": - value = metric["Percent"] if "Percent" in metric else metric["Rpm"] results_per_frame[frame_index]["fan_speed_rpm"] += ( - value / num_loops + metric["Rpm"] / num_loops ) # Filter out unavailable data and the timestamp elif metric is not None and metric_name != "timestamp": @@ -72,10 +69,10 @@ def output_top_passes(input_1, input_2): combined_data.index.names = ["Pass Name"] combined_data["pct_diff"] = ( combined_data["Input 1"] / combined_data["Input 2"] - ).abs().drop(metric_names(), errors="ignore") * 100 + ).abs().drop(metric_names()) * 100 combined_data = ( combined_data.sort_values(by="pct_diff", ascending=False) - #.iloc[:20] + .iloc[:20] .round() .dropna() .astype(int) @@ -100,10 +97,10 @@ def output_top_stdev(input_1, input_2): # depending on the set language combined_data["pct_diff"] = ( combined_data["Input 1"] / combined_data["Input 2"] - ).abs().drop(metric_names(), errors="ignore") * 100 + ).abs().drop(metric_names()) * 100 combined_data = ( combined_data.sort_values(by="pct_diff", ascending=False) - #.iloc[:20] + .iloc[:20] .round() .dropna() .astype(int) @@ -143,7 +140,7 @@ def main(): parser.print_usage() exit(2) - if not args.pass_mean_comparison and not args.pass_stdev_comparison: + if args.pass_mean_comparison is None and args.pass_stdev_comparison is None: print("Error: No analysis option specified, specify at least one\n") parser.print_usage() exit(1) diff --git a/scripts/deep_analysis_example.ipynb b/scripts/deep_analysis_example.ipynb index 45dbffd..6a3b799 100644 --- a/scripts/deep_analysis_example.ipynb +++ b/scripts/deep_analysis_example.ipynb @@ -229,198 +229,6 @@ " title=graph_name\n", " )" ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a27cb615-711d-47b7-b570-9fd2da3efb2f", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "0d18b49a-bb26-4dac-8c8d-f50f6d70fa3d", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "06b1a356-48f5-4a26-bfc6-0d34cdeaf3a4", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "0936c27b-d76a-4d2b-8acf-a7f9cfd957c9", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "bc7b1605-197a-4af5-928f-a59621265541", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e7d880f6-fbe6-4c82-9669-d55371add589", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "86ed4d62-2fbd-463b-bc98-8d25620dce33", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8f2ed3de-a804-48f8-bbd9-a0e9aa1c1461", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "33e1c5a6-6494-4915-9138-7e005b13e784", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5bbcc90f-4d88-44a0-9941-aa11bc400ebb", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "45778568-9d34-4ad8-8122-546f171e0e9a", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b5d46180-fa10-43ca-87ea-085321bd59ec", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3e563c23-2991-4b6b-9c60-49f497e02474", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8090f30c-6053-46b9-b110-ee0ca7748b1e", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a7fec099-b121-43f2-8ef6-ee7ef89ba690", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6d715819-9d82-42d0-9ee1-0766661e7e63", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e2c91a7f-5948-4446-b995-6f7b301f618d", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5e8a505b-4a73-4e6a-bc28-4e0f3adfbbdd", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "28bd4976-bb47-4287-858a-17449717f2d2", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ffa75d91-f087-47e4-abee-a9335ba1f81a", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7f84c673-e6d9-4df0-b911-f3d29119c8f7", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "28d8ec51-da05-49ca-bb9e-e6644d2fc3a9", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "178eb511-0039-4a72-97f2-02c51d267036", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e1f27031-080e-49bb-b1a4-91b3bd9fa0da", - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { From 1f67d42ea5819e7e7d6d5cb3a97be33434825bc5 Mon Sep 17 00:00:00 2001 From: rosaliedewinther Date: Mon, 17 Mar 2025 17:47:43 +0100 Subject: [PATCH 04/10] add gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..01746cb --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +scripts/.ipynb_checkpoints/ \ No newline at end of file From a03023edf577e3b1e991189f935e6baf9d7984aa Mon Sep 17 00:00:00 2001 From: rosaliedewinther Date: Mon, 17 Mar 2025 17:52:07 +0100 Subject: [PATCH 05/10] last fixes --- scripts/deep_analysis_example.ipynb | 32 +++++++++++++---------------- 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/scripts/deep_analysis_example.ipynb b/scripts/deep_analysis_example.ipynb index 6a3b799..13fcb90 100644 --- a/scripts/deep_analysis_example.ipynb +++ b/scripts/deep_analysis_example.ipynb @@ -8,7 +8,10 @@ "outputs": [], "source": [ "%%cmd\n", - "python -m pip install flatten_json" + "python -m pip install flatten_json\n", + "python -m pip install pandas\n", + "python -m pip install numpy\n", + "python -m pip install matplotlib" ] }, { @@ -41,7 +44,6 @@ " num_loops = len(json)\n", " for loop_results in json:\n", " for frame_index, frame_results in enumerate(loop_results[\"per_frame_results\"]):\n", - " #pprint.pprint(frame_results)\n", " if frame_index >= len(results_per_frame):\n", " results_per_frame.append(defaultdict(int))\n", " results_per_frame[frame_index]['sequence_time_ns'] = frame_results['sequence_time_ns']\n", @@ -66,21 +68,7 @@ " elif metric is not None and metric_name != \"timestamp\":\n", " results_per_frame[frame_index][metric_name] += metric / num_loops\n", " # TODO: Aggregate CPU timings\n", - " return pd.DataFrame([flatten(x) for x in results_per_frame])\n", - "\n", - "\n", - "def metric_names():\n", - " return [\n", - " \"edge_temperature_in_c\",\n", - " \"hotspot_temperature_in_c\",\n", - " \"usage_percentage\",\n", - " \"fan_speed_rpm\",\n", - " \"clock_speed_in_mhz\",\n", - " \"vram_clock_speed_in_mhz\",\n", - " \"board_power_usage_in_w\",\n", - " \"voltage_in_mv\",\n", - " \"vram_usage_in_mb\",\n", - " ]" + " return pd.DataFrame([flatten(x) for x in results_per_frame])" ] }, { @@ -208,7 +196,7 @@ "for graph_name in metrics['level_1'].unique():\n", " # Grab the metric we want to plot\n", " selected_metric = metrics[metrics['level_1'] == graph_name]\n", - " selected_metric = filtered_data.drop('level_1', axis=1)\n", + " selected_metric = selected_metric.drop('level_1', axis=1)\n", "\n", " # Filter outliers out of view\n", " max_mean = selected_metric[files].mean().mean()\n", @@ -229,6 +217,14 @@ " title=graph_name\n", " )" ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1ea8273f-13f5-4cc9-82c9-9fe3cf6dfeac", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { From f6e9473a0c53bd5730a7e212574930f3296cfb53 Mon Sep 17 00:00:00 2001 From: rosaliedewinther Date: Mon, 17 Mar 2025 17:52:41 +0100 Subject: [PATCH 06/10] newline to gitignore --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 01746cb..438f0f8 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1 @@ -scripts/.ipynb_checkpoints/ \ No newline at end of file +scripts/.ipynb_checkpoints/ From 2c4c3d75a4560acfb007825be154efe9530723f3 Mon Sep 17 00:00:00 2001 From: rosaliedewinther Date: Tue, 18 Mar 2025 13:30:04 +0100 Subject: [PATCH 07/10] split into 3 notebooks --- ...mple.ipynb => deep_analysis_plotter.ipynb} | 125 ++++++------------ scripts/per_frame_plotter.ipynb | 122 +++++++++++++++++ scripts/scores_plotter.ipynb | 91 +++++++++++++ 3 files changed, 250 insertions(+), 88 deletions(-) rename scripts/{deep_analysis_example.ipynb => deep_analysis_plotter.ipynb} (75%) create mode 100644 scripts/per_frame_plotter.ipynb create mode 100644 scripts/scores_plotter.ipynb diff --git a/scripts/deep_analysis_example.ipynb b/scripts/deep_analysis_plotter.ipynb similarity index 75% rename from scripts/deep_analysis_example.ipynb rename to scripts/deep_analysis_plotter.ipynb index 13fcb90..04abce6 100644 --- a/scripts/deep_analysis_example.ipynb +++ b/scripts/deep_analysis_plotter.ipynb @@ -9,9 +9,7 @@ "source": [ "%%cmd\n", "python -m pip install flatten_json\n", - "python -m pip install pandas\n", - "python -m pip install numpy\n", - "python -m pip install matplotlib" + "python -m pip install pandas" ] }, { @@ -24,18 +22,28 @@ "import pandas as pd\n", "import json\n", "from collections import defaultdict\n", - "import numpy as np\n", - "import argparse\n", - "import sys\n", - "from flatten_json import flatten\n", - "import matplotlib.pyplot as plt\n", - "import math" + "from flatten_json import flatten" ] }, { "cell_type": "code", "execution_count": null, - "id": "099d7d0d-febc-4562-801a-767c4b028f56", + "id": "59b6a968-ad86-48a0-8948-d1f42bf0c3c6", + "metadata": {}, + "outputs": [], + "source": [ + "files = [\n", + " '../../a_ray_tracing_inline_deep.json',\n", + " '../../b_ray_tracing_inline_deep.json',\n", + " '../../a_ray_tracing_pipeline_deep.json',\n", + " '../../b_ray_tracing_pipeline_deep.json',\n", + "]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a60a4c63-b884-4558-98bd-ee3d14bbc569", "metadata": {}, "outputs": [], "source": [ @@ -68,71 +76,32 @@ " elif metric is not None and metric_name != \"timestamp\":\n", " results_per_frame[frame_index][metric_name] += metric / num_loops\n", " # TODO: Aggregate CPU timings\n", - " return pd.DataFrame([flatten(x) for x in results_per_frame])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "939b04cb-7ffd-4963-8dac-71c0a76275cf", - "metadata": {}, - "outputs": [], - "source": [ - "files = [\n", - " '../../a_ray_tracing_inline.csv',\n", - " '../../a_ray_tracing_pipeline.csv',\n", - " '../../b_ray_tracing_inline.csv',\n", - " '../../b_ray_tracing_pipeline.csv',\n", - "]\n", - "\n", - "scores = {}\n", - "\n", - "for file in files:\n", - " # Read csv\n", - " file_scores = pd.read_csv(file)\n", - " # Remove empty results\n", - " file_scores = file_scores.loc[:, (file_scores != 0).any(axis=0)]\n", - " scores[file] = file_scores\n", - "scores = pd.concat(scores).groupby(level=0, sort=False).mean().T.drop('Loop', errors=\"ignore\")\n", - "scores" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2fbaaf11-5b32-43ed-84d1-8894ec6de013", - "metadata": {}, - "outputs": [], - "source": [ - "scores.plot(kind=\"bar\", figsize=(20,5), colormap='Dark2', grid=True, rot=0)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a60a4c63-b884-4558-98bd-ee3d14bbc569", - "metadata": {}, - "outputs": [], - "source": [ - "files = [\n", - " '../../a_ray_tracing_inline_deep.json',\n", - " '../../b_ray_tracing_inline_deep.json',\n", - " '../../a_ray_tracing_pipeline_deep.json',\n", - " '../../b_ray_tracing_pipeline_deep.json',\n", - "]\n", + " return pd.DataFrame([flatten(x) for x in results_per_frame])\n", "\n", "results = {}\n", "\n", "# Load all files into one large dataframe\n", "for path in files:\n", " with open(path, \"r\") as json_file:\n", - " # We agregate passes within each frame, so we get one number per pass per frame per input file\n", + " # We aggregate passes within each frame, so we get one number per pass per frame per input file\n", " json_data = aggregate_loops_passes(json.load(json_file))\n", " results[path] = json_data\n", + "# Concat into input file + sequence time per row, metric per column\n", "full_dataset = pd.concat(results)\n", "full_dataset" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "7de3e2b9-0470-40a2-9ed7-c2d931aaf153", + "metadata": {}, + "outputs": [], + "source": [ + "# Print all possible metrics\n", + "full_dataset.columns.tolist()" + ] + }, { "cell_type": "code", "execution_count": null, @@ -164,28 +133,16 @@ "relevant_metrics_with_time = relevant_metrics + [\"sequence_time_ns\"]\n", "metrics = full_dataset[relevant_metrics_with_time]\n", "\n", - "# Use sequence time and input name as index, each row will have a unique time and input\n", + "# Reshape into sequence time + metric type per row, input file per column\n", "metrics = metrics.reset_index().set_index(['sequence_time_ns', 'level_0']).drop('level_1', axis=1)\n", - "\n", - "# Make each input file a column, and put the pass names into a specific column\n", "metrics = metrics.stack().unstack(1).reset_index()\n", "\n", - "# From nano seconds to seconds\n", - "metrics['sequence_time_ns'] = metrics['sequence_time_ns'] / 1_000_000_000\n", + "# From ns to s\n", + "metrics['sequence_time_s'] = metrics['sequence_time_ns'] / 1_000_000_000\n", + "metrics = metrics.drop('sequence_time_ns', axis=1)\n", "metrics" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "217b62b6-2500-45b5-b3f0-24449387fddd", - "metadata": {}, - "outputs": [], - "source": [ - "# Print all possible metrics\n", - "full_dataset.columns.tolist()" - ] - }, { "cell_type": "code", "execution_count": null, @@ -206,7 +163,7 @@ "\n", " # Plot results \n", " selected_metric.infer_objects(copy=False).interpolate(method='linear').plot(\n", - " x='sequence_time_ns', \n", + " x='sequence_time_s', \n", " ylabel='shader execution time in ms',\n", " xlabel='benchmark timeline in seconds', \n", " ylim= (max(0, max_mean - max_std * 3), max_mean + max_std * 3),\n", @@ -217,14 +174,6 @@ " title=graph_name\n", " )" ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1ea8273f-13f5-4cc9-82c9-9fe3cf6dfeac", - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { diff --git a/scripts/per_frame_plotter.ipynb b/scripts/per_frame_plotter.ipynb new file mode 100644 index 0000000..6b24c39 --- /dev/null +++ b/scripts/per_frame_plotter.ipynb @@ -0,0 +1,122 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "1d9d9451-25fe-40d7-a67e-2fcaf6a9f54a", + "metadata": {}, + "outputs": [], + "source": [ + "%%cmd\n", + "python -m pip install pandas" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4a5f8126-43bc-4cba-b65b-136cbf8971ab", + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "959dce8e-4f96-4505-ba35-0300c3aee0a1", + "metadata": {}, + "outputs": [], + "source": [ + "files = [\n", + " '../../a_ray_tracing_inline_perframe.csv',\n", + " '../../a_ray_tracing_pipeline_perframe.csv',\n", + " '../../b_ray_tracing_inline_perframe.csv',\n", + " '../../b_ray_tracing_pipeline_perframe.csv',\n", + "]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a5f9ea81-7bd2-4f70-a0b8-45164afe8f76", + "metadata": {}, + "outputs": [], + "source": [ + "scores = {}\n", + "\n", + "for file in files:\n", + " # Read csv\n", + " file_scores = pd.read_csv(file)\n", + " # Remove empty results\n", + " file_scores = file_scores.loc[:, (file_scores != 0).any(axis=0)]\n", + " scores[file] = file_scores\n", + "# Reshape into row per input file + sequence time, column per metric\n", + "scores = pd.concat(scores).reset_index().set_index(['level_0', 'Sequence Time (ns)']).drop(['level_1', 'Loop Index', 'Frame'], axis=1)\n", + "scores" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b4bbc8ad-fbf4-4c96-8aa4-afb02ff305a0", + "metadata": {}, + "outputs": [], + "source": [ + "# Reshape into row per sequence time + metric, column per input file\n", + "new_scores = scores.stack().unstack(0).reset_index()\n", + "# ns to s for the sequence time\n", + "new_scores['Sequence Time (s)'] = new_scores['Sequence Time (ns)'] / 1_000_000_000\n", + "new_scores = new_scores.drop('Sequence Time (ns)', axis=1)\n", + "new_scores" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2d6b6f75-251b-4bb2-aa44-e6393cf8140a", + "metadata": {}, + "outputs": [], + "source": [ + "for graph_name in new_scores['level_1'].unique():\n", + " graph_data = new_scores[new_scores['level_1'] == graph_name]\n", + " graph_data = graph_data.drop('level_1', axis=1)\n", + " graph_data = graph_data.apply(pd.to_numeric, errors='coerce').interpolate(method='linear')\n", + " \n", + " graph_data.plot(\n", + " x='Sequence Time (s)', \n", + " ylabel='value',\n", + " xlabel='benchmark timeline in seconds', \n", + " kind=\"line\", \n", + " logy=True,\n", + " figsize=(20,5), \n", + " colormap='Dark2', \n", + " grid=True, \n", + " rot=0,\n", + " title=graph_name\n", + " )" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.4" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/scripts/scores_plotter.ipynb b/scripts/scores_plotter.ipynb new file mode 100644 index 0000000..9429986 --- /dev/null +++ b/scripts/scores_plotter.ipynb @@ -0,0 +1,91 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "c7a3c3bd-54d5-4979-8502-ac69a32a13fb", + "metadata": {}, + "outputs": [], + "source": [ + "%%cmd\n", + "python -m pip install pandas" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "21658370-184d-4ab0-b210-d8ab93f0818b", + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "07bb93e8-5d29-4bbf-8f44-be5db0af70c7", + "metadata": {}, + "outputs": [], + "source": [ + "files = [\n", + " '../../a_ray_tracing_inline.csv',\n", + " '../../a_ray_tracing_pipeline.csv',\n", + " '../../b_ray_tracing_inline.csv',\n", + " '../../b_ray_tracing_pipeline.csv',\n", + "]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9088c71c-0fe8-4c87-852e-265b662dcacc", + "metadata": {}, + "outputs": [], + "source": [ + "scores = {}\n", + "\n", + "for file in files:\n", + " # Read csv\n", + " file_scores = pd.read_csv(file)\n", + " # Remove empty results\n", + " file_scores = file_scores.loc[:, (file_scores != 0).any(axis=0)]\n", + " scores[file] = file_scores\n", + "# Reshape into row per metric, column per input file\n", + "scores = pd.concat(scores).groupby(level=0, sort=False).mean().T.drop('Loop', errors=\"ignore\")\n", + "scores" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d89bab16-7553-4fd2-953b-05a054362999", + "metadata": {}, + "outputs": [], + "source": [ + "scores.plot(kind=\"bar\", figsize=(20,5), colormap='Dark2', grid=True, rot=0)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.4" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From adda78ca8304d7df2eac85530f456b38a2fb228f Mon Sep 17 00:00:00 2001 From: rosaliedewinther Date: Tue, 18 Mar 2025 13:31:28 +0100 Subject: [PATCH 08/10] use local filenames as example --- scripts/deep_analysis_plotter.ipynb | 8 ++++---- scripts/per_frame_plotter.ipynb | 8 ++++---- scripts/scores_plotter.ipynb | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/scripts/deep_analysis_plotter.ipynb b/scripts/deep_analysis_plotter.ipynb index 04abce6..d9165a9 100644 --- a/scripts/deep_analysis_plotter.ipynb +++ b/scripts/deep_analysis_plotter.ipynb @@ -33,10 +33,10 @@ "outputs": [], "source": [ "files = [\n", - " '../../a_ray_tracing_inline_deep.json',\n", - " '../../b_ray_tracing_inline_deep.json',\n", - " '../../a_ray_tracing_pipeline_deep.json',\n", - " '../../b_ray_tracing_pipeline_deep.json',\n", + " 'a_inline_deep.json',\n", + " 'b_inline_deep.json',\n", + " 'a_pipeline_deep.json',\n", + " 'b_pipeline_deep.json',\n", "]" ] }, diff --git a/scripts/per_frame_plotter.ipynb b/scripts/per_frame_plotter.ipynb index 6b24c39..62bc696 100644 --- a/scripts/per_frame_plotter.ipynb +++ b/scripts/per_frame_plotter.ipynb @@ -29,10 +29,10 @@ "outputs": [], "source": [ "files = [\n", - " '../../a_ray_tracing_inline_perframe.csv',\n", - " '../../a_ray_tracing_pipeline_perframe.csv',\n", - " '../../b_ray_tracing_inline_perframe.csv',\n", - " '../../b_ray_tracing_pipeline_perframe.csv',\n", + " 'a_inline_perframe.csv',\n", + " 'a_pipeline_perframe.csv',\n", + " 'b_inline_perframe.csv',\n", + " 'b_pipeline_perframe.csv',\n", "]" ] }, diff --git a/scripts/scores_plotter.ipynb b/scripts/scores_plotter.ipynb index 9429986..80ef4a5 100644 --- a/scripts/scores_plotter.ipynb +++ b/scripts/scores_plotter.ipynb @@ -29,10 +29,10 @@ "outputs": [], "source": [ "files = [\n", - " '../../a_ray_tracing_inline.csv',\n", - " '../../a_ray_tracing_pipeline.csv',\n", - " '../../b_ray_tracing_inline.csv',\n", - " '../../b_ray_tracing_pipeline.csv',\n", + " 'a_inline.csv',\n", + " 'a_pipeline.csv',\n", + " 'b_inline.csv',\n", + " 'b_pipeline.csv',\n", "]" ] }, From 6b2f73cf90208b3fc98b0beb93267fe8ed387288 Mon Sep 17 00:00:00 2001 From: Max de Danschutter Date: Wed, 19 Mar 2025 14:58:35 +0100 Subject: [PATCH 09/10] update scores plotter --- scripts/scores_plotter.ipynb | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/scripts/scores_plotter.ipynb b/scripts/scores_plotter.ipynb index 80ef4a5..b117475 100644 --- a/scripts/scores_plotter.ipynb +++ b/scripts/scores_plotter.ipynb @@ -8,7 +8,8 @@ "outputs": [], "source": [ "%%cmd\n", - "python -m pip install pandas" + "python -m pip install pandas\n", + "python -m pip install matplotlib" ] }, { @@ -29,10 +30,10 @@ "outputs": [], "source": [ "files = [\n", - " 'a_inline.csv',\n", - " 'a_pipeline.csv',\n", - " 'b_inline.csv',\n", - " 'b_pipeline.csv',\n", + " 'scores/5070ti_inline.csv',\n", + " 'scores/5070ti_pipeline.csv',\n", + " 'scores/9070xt_inline.csv',\n", + " 'scores/9070xt_pipeline.csv',\n", "]" ] }, @@ -63,7 +64,7 @@ "metadata": {}, "outputs": [], "source": [ - "scores.plot(kind=\"bar\", figsize=(20,5), colormap='Dark2', grid=True, rot=0)" + "scores.plot(kind=\"bar\", figsize=(20,5), colormap='Dark2', grid=True, rot=0, title=\"Evolve scores (higher is better)\", ylabel=\"score\")" ] } ], @@ -83,7 +84,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.4" + "version": "3.13.2" } }, "nbformat": 4, From de99d355ab8741d7f6ec3ccf507242dca6f52359 Mon Sep 17 00:00:00 2001 From: Max de Danschutter Date: Wed, 19 Mar 2025 15:31:01 +0100 Subject: [PATCH 10/10] remove `scores/` prefix and `.csv` postfix from score names --- scripts/scores_plotter.ipynb | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/scores_plotter.ipynb b/scripts/scores_plotter.ipynb index b117475..ec4ced0 100644 --- a/scripts/scores_plotter.ipynb +++ b/scripts/scores_plotter.ipynb @@ -19,7 +19,8 @@ "metadata": {}, "outputs": [], "source": [ - "import pandas as pd" + "import pandas as pd\n", + "import os" ] }, { @@ -51,7 +52,9 @@ " file_scores = pd.read_csv(file)\n", " # Remove empty results\n", " file_scores = file_scores.loc[:, (file_scores != 0).any(axis=0)]\n", - " scores[file] = file_scores\n", + " # Use os.path.basename() to get only the file name\n", + " file_name = os.path.splitext(os.path.basename(file))[0]\n", + " scores[file_name] = file_scores\n", "# Reshape into row per metric, column per input file\n", "scores = pd.concat(scores).groupby(level=0, sort=False).mean().T.drop('Loop', errors=\"ignore\")\n", "scores"