diff --git a/scripts/deep_analysis_plotter.ipynb b/scripts/deep_analysis_plotter.ipynb index d9165a9..7010537 100644 --- a/scripts/deep_analysis_plotter.ipynb +++ b/scripts/deep_analysis_plotter.ipynb @@ -9,12 +9,13 @@ "source": [ "%%cmd\n", "python -m pip install flatten_json\n", - "python -m pip install pandas" + "python -m pip install pandas\n", + "python -m pip install matplotlib" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 24, "id": "bbdc923a-44fd-4250-990b-80771446bbf0", "metadata": {}, "outputs": [], @@ -22,7 +23,9 @@ "import pandas as pd\n", "import json\n", "from collections import defaultdict\n", - "from flatten_json import flatten" + "from flatten_json import flatten\n", + "import matplotlib.pyplot as plt\n", + "import pathlib" ] }, { @@ -65,16 +68,11 @@ " results_per_frame[frame_index][scope_name] += (\n", " scope_timing[\"end\"] - scope_timing[\"start\"]\n", " ) / num_loops / 1_000_000 # in ms\n", - " for metric_name, metric in frame_results[\"metrics\"].items():\n", - " # TODO: Flatten this in rust to fan_speed_rpm\n", - " if metric_name == \"fan_speed\":\n", - " value = metric[\"Percent\"] if \"Percent\" in metric else metric[\"Rpm\"]\n", - " results_per_frame[frame_index][\"fan_speed_rpm\"] += (\n", - " value / num_loops\n", - " )\n", - " # Filter out unavailable data and the timestamp\n", - " elif metric is not None and metric_name != \"timestamp\":\n", - " results_per_frame[frame_index][metric_name] += metric / num_loops\n", + " if frame_results[\"metrics\"] is not None:\n", + " for metric_name, metric in frame_results[\"metrics\"].items():\n", + " # TODO: Flatten this in rust to fan_speed_rpm\n", + " if metric is not None and metric_name != \"timestamp\":\n", + " results_per_frame[frame_index][metric_name] += metric / num_loops\n", " # TODO: Aggregate CPU timings\n", " return pd.DataFrame([flatten(x) for x in results_per_frame])\n", "\n", @@ -109,29 +107,7 @@ "metadata": {}, "outputs": [], "source": [ - "relevant_metrics = [\n", - " 'reflection-hits-shading',\n", - " 'water-compositing',\n", - " 'blur',\n", - " 'diffuse-spatial-filter',\n", - " 'spd',\n", - " 'sun-direct-lighting',\n", - " 'reflection-ray-tracing-inline',\n", - " 'trace-diffuse-nee-rays',\n", - " 'render pass',\n", - " 'shadow-ray-tracing-pipeline',\n", - " 'compositing',\n", - " 'build-gbuffers_0',\n", - " 'scale-raster',\n", - " 'Batch refit bottom level',\n", - " 'clock_speed_in_mhz',\n", - " 'board_power_usage_in_w',\n", - " 'vram_usage_in_mb',\n", - " 'edge_temperature_in_c'\n", - "]\n", - "# We want all relevant metrics with the sequence time, to properly plot on the x axis\n", - "relevant_metrics_with_time = relevant_metrics + [\"sequence_time_ns\"]\n", - "metrics = full_dataset[relevant_metrics_with_time]\n", + "metrics = full_dataset\n", "\n", "# Reshape into sequence time + metric type per row, input file per column\n", "metrics = metrics.reset_index().set_index(['sequence_time_ns', 'level_0']).drop('level_1', axis=1)\n", @@ -150,37 +126,34 @@ "metadata": {}, "outputs": [], "source": [ + "pathlib.Path('output_analysis').mkdir(parents=True, exist_ok=True) \n", "for graph_name in metrics['level_1'].unique():\n", " # Grab the metric we want to plot\n", " selected_metric = metrics[metrics['level_1'] == graph_name]\n", " selected_metric = selected_metric.drop('level_1', axis=1)\n", "\n", - " # Filter outliers out of view\n", - " max_mean = selected_metric[files].mean().mean()\n", - " max_mean = 0 if pd.isna(max_mean) else max_mean\n", - " max_std = selected_metric[files].std(axis=1).max()\n", - " max_std = selected_metric[files].max() / 3.0 if pd.isna(max_std) else max_std\n", "\n", " # Plot results \n", " selected_metric.infer_objects(copy=False).interpolate(method='linear').plot(\n", " x='sequence_time_s', \n", " ylabel='shader execution time in ms',\n", " xlabel='benchmark timeline in seconds', \n", - " ylim= (max(0, max_mean - max_std * 3), max_mean + max_std * 3),\n", + " #ylim= (max(0, max_mean - max_std * 3), max_mean + max_std * 3),\n", " figsize=(20,10), \n", " colormap='Dark2', \n", " grid=True, \n", " legend=True,\n", " title=graph_name\n", - " )" + " )\n", + " plt.savefig('output_analysis/{}.png'.format(graph_name))" ] } ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "Python [conda env:base] *", "language": "python", - "name": "python3" + "name": "conda-base-py" }, "language_info": { "codemirror_mode": { @@ -192,7 +165,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.4" + "version": "3.13.5" } }, "nbformat": 4, diff --git a/scripts/per_frame_plotter.ipynb b/scripts/per_frame_plotter.ipynb index 62bc696..65249a5 100644 --- a/scripts/per_frame_plotter.ipynb +++ b/scripts/per_frame_plotter.ipynb @@ -18,7 +18,10 @@ "metadata": {}, "outputs": [], "source": [ - "import pandas as pd" + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "import pathlib\n", + "pd.set_option('display.float_format', lambda x: '%.3f' % x)" ] }, { @@ -51,8 +54,24 @@ " # Remove empty results\n", " file_scores = file_scores.loc[:, (file_scores != 0).any(axis=0)]\n", " scores[file] = file_scores\n", + "scores = pd.concat(scores)\n", + "scores['Frame time (ms)'] = (scores['Sequence Time (ns)'].shift(-1) - scores['Sequence Time (ns)']) / 1_000_000\n", + "# The final frame always has a negative frame time, probably because it loops\n", + "# The first 3 frames are not representative, and are huge outliers\n", + "scores['Frame time (ms)'] = scores['Frame time (ms)'].mask(scores['Frame time (ms)'] < 0)\n", + "scores['Frame time (ms)'] = scores['Frame time (ms)'].mask(scores['Frame'] < 3)\n", "# Reshape into row per input file + sequence time, column per metric\n", - "scores = pd.concat(scores).reset_index().set_index(['level_0', 'Sequence Time (ns)']).drop(['level_1', 'Loop Index', 'Frame'], axis=1)\n", + "scores = scores.reset_index().set_index(['level_0', 'Sequence Time (ns)']).drop(['level_1', 'Loop Index', 'Frame'], axis=1)\n", + "scores['Ray Tracing (ms)'] = scores['Ray Tracing (ns)'] / 1_000_000\n", + "scores = scores.drop('Ray Tracing (ns)', axis=1)\n", + "scores['Acceleration Structure Build (ms)'] = scores['Acceleration Structure Build (ns)'] / 1_000_000\n", + "scores = scores.drop('Acceleration Structure Build (ns)', axis=1)\n", + "scores['Rasterization (ms)'] = scores['Rasterization (ns)'] / 1_000_000\n", + "scores = scores.drop('Rasterization (ns)', axis=1)\n", + "scores['Compute (ms)'] = scores['Compute (ns)'] / 1_000_000\n", + "scores = scores.drop('Compute (ns)', axis=1)\n", + "scores['Driver (ms)'] = scores['Driver (ns)'] / 1_000_000\n", + "scores = scores.drop('Driver (ns)', axis=1)\n", "scores" ] }, @@ -78,6 +97,7 @@ "metadata": {}, "outputs": [], "source": [ + "pathlib.Path('output_per_frame').mkdir(parents=True, exist_ok=True) \n", "for graph_name in new_scores['level_1'].unique():\n", " graph_data = new_scores[new_scores['level_1'] == graph_name]\n", " graph_data = graph_data.drop('level_1', axis=1)\n", @@ -88,21 +108,22 @@ " ylabel='value',\n", " xlabel='benchmark timeline in seconds', \n", " kind=\"line\", \n", - " logy=True,\n", + " logy=False,\n", " figsize=(20,5), \n", " colormap='Dark2', \n", " grid=True, \n", " rot=0,\n", " title=graph_name\n", - " )" + " )\n", + " plt.savefig('output_per_frame/{}.png'.format(graph_name))" ] } ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "Python [conda env:base] *", "language": "python", - "name": "python3" + "name": "conda-base-py" }, "language_info": { "codemirror_mode": { @@ -114,7 +135,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.4" + "version": "3.13.5" } }, "nbformat": 4,