Skip to content

🦫 Further improvement to graphs and write out results to png's #6

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
jasperdewinther wants to merge 2 commits into
base: improved-graphs
Choose a base branch
from
Open

🦫 Further improvement to graphs and write out results to png's #6

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8fdfa2b
improved further
jasperdewinther Oct 17, 2025
4e7e979
add missing import and check if metrics exists
jasperdewinther Nov 12, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
65 changes: 19 additions & 46 deletions scripts/deep_analysis_plotter.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,20 +9,23 @@
"source": [
"%%cmd\n",
"python -m pip install flatten_json\n",
"python -m pip install pandas"
"python -m pip install pandas\n",
"python -m pip install matplotlib"
]
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 24,
"id": "bbdc923a-44fd-4250-990b-80771446bbf0",
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"import json\n",
"from collections import defaultdict\n",
"from flatten_json import flatten"
"from flatten_json import flatten\n",
"import matplotlib.pyplot as plt\n",
"import pathlib"
]
},
{
Expand Down Expand Up @@ -65,16 +68,11 @@
" results_per_frame[frame_index][scope_name] += (\n",
" scope_timing[\"end\"] - scope_timing[\"start\"]\n",
" ) / num_loops / 1_000_000 # in ms\n",
" for metric_name, metric in frame_results[\"metrics\"].items():\n",
" # TODO: Flatten this in rust to fan_speed_rpm\n",
" if metric_name == \"fan_speed\":\n",
" value = metric[\"Percent\"] if \"Percent\" in metric else metric[\"Rpm\"]\n",
" results_per_frame[frame_index][\"fan_speed_rpm\"] += (\n",
" value / num_loops\n",
" )\n",
" # Filter out unavailable data and the timestamp\n",
" elif metric is not None and metric_name != \"timestamp\":\n",
" results_per_frame[frame_index][metric_name] += metric / num_loops\n",
" if frame_results[\"metrics\"] is not None:\n",
" for metric_name, metric in frame_results[\"metrics\"].items():\n",
" # TODO: Flatten this in rust to fan_speed_rpm\n",
" if metric is not None and metric_name != \"timestamp\":\n",
" results_per_frame[frame_index][metric_name] += metric / num_loops\n",
" # TODO: Aggregate CPU timings\n",
" return pd.DataFrame([flatten(x) for x in results_per_frame])\n",
"\n",
Expand Down Expand Up @@ -109,29 +107,7 @@
"metadata": {},
"outputs": [],
"source": [
"relevant_metrics = [\n",
" 'reflection-hits-shading',\n",
" 'water-compositing',\n",
" 'blur',\n",
" 'diffuse-spatial-filter',\n",
" 'spd',\n",
" 'sun-direct-lighting',\n",
" 'reflection-ray-tracing-inline',\n",
" 'trace-diffuse-nee-rays',\n",
" 'render pass',\n",
" 'shadow-ray-tracing-pipeline',\n",
" 'compositing',\n",
" 'build-gbuffers_0',\n",
" 'scale-raster',\n",
" 'Batch refit bottom level',\n",
" 'clock_speed_in_mhz',\n",
" 'board_power_usage_in_w',\n",
" 'vram_usage_in_mb',\n",
" 'edge_temperature_in_c'\n",
"]\n",
"# We want all relevant metrics with the sequence time, to properly plot on the x axis\n",
"relevant_metrics_with_time = relevant_metrics + [\"sequence_time_ns\"]\n",
"metrics = full_dataset[relevant_metrics_with_time]\n",
"metrics = full_dataset\n",
"\n",
"# Reshape into sequence time + metric type per row, input file per column\n",
"metrics = metrics.reset_index().set_index(['sequence_time_ns', 'level_0']).drop('level_1', axis=1)\n",
Expand All @@ -150,37 +126,34 @@
"metadata": {},
"outputs": [],
"source": [
"pathlib.Path('output_analysis').mkdir(parents=True, exist_ok=True) \n",
"for graph_name in metrics['level_1'].unique():\n",
" # Grab the metric we want to plot\n",
" selected_metric = metrics[metrics['level_1'] == graph_name]\n",
" selected_metric = selected_metric.drop('level_1', axis=1)\n",
"\n",
" # Filter outliers out of view\n",
" max_mean = selected_metric[files].mean().mean()\n",
" max_mean = 0 if pd.isna(max_mean) else max_mean\n",
" max_std = selected_metric[files].std(axis=1).max()\n",
" max_std = selected_metric[files].max() / 3.0 if pd.isna(max_std) else max_std\n",
"\n",
" # Plot results \n",
" selected_metric.infer_objects(copy=False).interpolate(method='linear').plot(\n",
" x='sequence_time_s', \n",
" ylabel='shader execution time in ms',\n",
" xlabel='benchmark timeline in seconds', \n",
" ylim= (max(0, max_mean - max_std * 3), max_mean + max_std * 3),\n",
" #ylim= (max(0, max_mean - max_std * 3), max_mean + max_std * 3),\n",
" figsize=(20,10), \n",
" colormap='Dark2', \n",
" grid=True, \n",
" legend=True,\n",
" title=graph_name\n",
" )"
" )\n",
" plt.savefig('output_analysis/{}.png'.format(graph_name))"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"display_name": "Python [conda env:base] *",
"language": "python",
"name": "python3"
"name": "conda-base-py"
},
"language_info": {
"codemirror_mode": {
Expand All @@ -192,7 +165,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.12.4"
"version": "3.13.5"
}
},
"nbformat": 4,
Expand Down
35 changes: 28 additions & 7 deletions scripts/per_frame_plotter.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,10 @@
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd"
"import pandas as pd\n",
"import matplotlib.pyplot as plt\n",
"import pathlib\n",
"pd.set_option('display.float_format', lambda x: '%.3f' % x)"
]
},
{
Expand Down Expand Up @@ -51,8 +54,24 @@
" # Remove empty results\n",
" file_scores = file_scores.loc[:, (file_scores != 0).any(axis=0)]\n",
" scores[file] = file_scores\n",
"scores = pd.concat(scores)\n",
"scores['Frame time (ms)'] = (scores['Sequence Time (ns)'].shift(-1) - scores['Sequence Time (ns)']) / 1_000_000\n",
"# The final frame always has a negative frame time, probably because it loops\n",
"# The first 3 frames are not representative, and are huge outliers\n",
"scores['Frame time (ms)'] = scores['Frame time (ms)'].mask(scores['Frame time (ms)'] < 0)\n",
"scores['Frame time (ms)'] = scores['Frame time (ms)'].mask(scores['Frame'] < 3)\n",
"# Reshape into row per input file + sequence time, column per metric\n",
"scores = pd.concat(scores).reset_index().set_index(['level_0', 'Sequence Time (ns)']).drop(['level_1', 'Loop Index', 'Frame'], axis=1)\n",
"scores = scores.reset_index().set_index(['level_0', 'Sequence Time (ns)']).drop(['level_1', 'Loop Index', 'Frame'], axis=1)\n",
"scores['Ray Tracing (ms)'] = scores['Ray Tracing (ns)'] / 1_000_000\n",
"scores = scores.drop('Ray Tracing (ns)', axis=1)\n",
"scores['Acceleration Structure Build (ms)'] = scores['Acceleration Structure Build (ns)'] / 1_000_000\n",
"scores = scores.drop('Acceleration Structure Build (ns)', axis=1)\n",
"scores['Rasterization (ms)'] = scores['Rasterization (ns)'] / 1_000_000\n",
"scores = scores.drop('Rasterization (ns)', axis=1)\n",
"scores['Compute (ms)'] = scores['Compute (ns)'] / 1_000_000\n",
"scores = scores.drop('Compute (ns)', axis=1)\n",
"scores['Driver (ms)'] = scores['Driver (ns)'] / 1_000_000\n",
"scores = scores.drop('Driver (ns)', axis=1)\n",
"scores"
]
},
Expand All @@ -78,6 +97,7 @@
"metadata": {},
"outputs": [],
"source": [
"pathlib.Path('output_per_frame').mkdir(parents=True, exist_ok=True) \n",
"for graph_name in new_scores['level_1'].unique():\n",
" graph_data = new_scores[new_scores['level_1'] == graph_name]\n",
" graph_data = graph_data.drop('level_1', axis=1)\n",
Expand All @@ -88,21 +108,22 @@
" ylabel='value',\n",
" xlabel='benchmark timeline in seconds', \n",
" kind=\"line\", \n",
" logy=True,\n",
" logy=False,\n",
" figsize=(20,5), \n",
" colormap='Dark2', \n",
" grid=True, \n",
" rot=0,\n",
" title=graph_name\n",
" )"
" )\n",
" plt.savefig('output_per_frame/{}.png'.format(graph_name))"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"display_name": "Python [conda env:base] *",
"language": "python",
"name": "python3"
"name": "conda-base-py"
},
"language_info": {
"codemirror_mode": {
Expand All @@ -114,7 +135,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.12.4"
"version": "3.13.5"
}
},
"nbformat": 4,
Expand Down