Note
Go to the end to download the full example code.
Measures the exporter success on many test cases¶
All test cases can be found in module
experimental_experiment.torch_interpreter.eval.model_cases.
Page l-exp-cases shows the exported
program for many of those cases.
from experimental_experiment.args import get_parsed_args
script_args = get_parsed_args(
"plot_exporter_coverage",
description=__doc__,
exporter=("all", "an exporter to rerun"),
dynamic=("all", "use dyanmic shapes"),
case=(
"three",
"model cases, two for the first two (to test), "
"all to select all, a name or a regular expression fior a subset",
),
quiet=("1", "0 or 1"),
verbose=("1", "verbosity"),
expose="exporter,dyanmic,case,quiet,verbose",
)
exporters = (
(
"export-strict",
"export-strict-dec",
"export-nostrict",
"export-nostrict-dec",
"export-jit",
"export-tracing",
"custom-strict",
"custom-nostrict",
"custom-strict-dec",
"custom-nostrict-dec",
"custom-tracing",
"dynamo",
"dynamo-ir",
"script",
)
if script_args.exporter == "all"
else script_args.exporter.split(",")
)
dynamic = (0, 1) if script_args.dynamic == "all" else (int(script_args.dynamic),)
cases = None if script_args.case == "all" else script_args.case.split(",")
quiet = bool(int(script_args.quiet))
verbose = int(script_args.verbose)
import pandas
from experimental_experiment.torch_interpreter.eval import evaluation
obs = evaluation(
exporters=exporters, dynamic=dynamic, cases=cases, quiet=quiet, verbose=verbose
)
0%| | 0/84 [00:00<?, ?it/s]
2%|▏ | 2/84 [00:00<00:07, 10.71it/s]
5%|▍ | 4/84 [00:00<00:06, 12.88it/s]
10%|▉ | 8/84 [00:00<00:03, 21.06it/s]
13%|█▎ | 11/84 [00:00<00:05, 12.95it/s]
15%|█▌ | 13/84 [00:01<00:14, 4.76it/s]
19%|█▉ | 16/84 [00:02<00:17, 3.85it/s]
21%|██▏ | 18/84 [00:03<00:14, 4.57it/s]
25%|██▌ | 21/84 [00:03<00:10, 6.18it/s]
27%|██▋ | 23/84 [00:03<00:10, 6.01it/s]
29%|██▊ | 24/84 [00:03<00:11, 5.44it/s]
31%|███ | 26/84 [00:04<00:13, 4.42it/s]
32%|███▏ | 27/84 [00:05<00:15, 3.60it/s]
36%|███▌ | 30/84 [00:05<00:10, 5.37it/s]
38%|███▊ | 32/84 [00:05<00:07, 6.51it/s]
43%|████▎ | 36/84 [00:05<00:04, 9.71it/s]
45%|████▌ | 38/84 [00:06<00:08, 5.22it/s]
48%|████▊ | 40/84 [00:07<00:09, 4.81it/s]
49%|████▉ | 41/84 [00:07<00:11, 3.59it/s]
51%|█████ | 43/84 [00:07<00:08, 4.60it/s]
52%|█████▏ | 44/84 [00:08<00:08, 4.49it/s]
55%|█████▍ | 46/84 [00:08<00:06, 5.56it/s]
58%|█████▊ | 49/84 [00:08<00:04, 7.98it/s]
61%|██████ | 51/84 [00:08<00:04, 7.71it/s]
63%|██████▎ | 53/84 [00:09<00:04, 6.96it/s]
64%|██████▍ | 54/84 [00:10<00:10, 2.97it/s]
65%|██████▌ | 55/84 [00:11<00:11, 2.42it/s]/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/export/_unlift.py:75: UserWarning: Attempted to insert a get_attr Node with no underlying reference in the owning GraphModule! Call GraphModule.add_submodule to add the necessary submodule, GraphModule.add_parameter to add the necessary Parameter, or nn.Module.register_buffer to add the necessary buffer
getattr_node = gm.graph.get_attr(lifted_node)
/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/fx/graph.py:1801: UserWarning: Node bias target bias bias of does not reference an nn.Module, nn.Parameter, or buffer, which is what 'get_attr' Nodes typically target
warnings.warn(
/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/export/_unlift.py:75: UserWarning: Attempted to insert a get_attr Node with no underlying reference in the owning GraphModule! Call GraphModule.add_submodule to add the necessary submodule, GraphModule.add_parameter to add the necessary Parameter, or nn.Module.register_buffer to add the necessary buffer
getattr_node = gm.graph.get_attr(lifted_node)
/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/fx/graph.py:1801: UserWarning: Node bias target bias bias of does not reference an nn.Module, nn.Parameter, or buffer, which is what 'get_attr' Nodes typically target
warnings.warn(
/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/export/_unlift.py:75: UserWarning: Attempted to insert a get_attr Node with no underlying reference in the owning GraphModule! Call GraphModule.add_submodule to add the necessary submodule, GraphModule.add_parameter to add the necessary Parameter, or nn.Module.register_buffer to add the necessary buffer
getattr_node = gm.graph.get_attr(lifted_node)
/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/fx/graph.py:1801: UserWarning: Node bias target bias bias of does not reference an nn.Module, nn.Parameter, or buffer, which is what 'get_attr' Nodes typically target
warnings.warn(
69%|██████▉ | 58/84 [00:11<00:07, 3.47it/s]/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/export/_unlift.py:75: UserWarning: Attempted to insert a get_attr Node with no underlying reference in the owning GraphModule! Call GraphModule.add_submodule to add the necessary submodule, GraphModule.add_parameter to add the necessary Parameter, or nn.Module.register_buffer to add the necessary buffer
getattr_node = gm.graph.get_attr(lifted_node)
/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/fx/graph.py:1801: UserWarning: Node bias target bias bias of does not reference an nn.Module, nn.Parameter, or buffer, which is what 'get_attr' Nodes typically target
warnings.warn(
/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/export/_unlift.py:75: UserWarning: Attempted to insert a get_attr Node with no underlying reference in the owning GraphModule! Call GraphModule.add_submodule to add the necessary submodule, GraphModule.add_parameter to add the necessary Parameter, or nn.Module.register_buffer to add the necessary buffer
getattr_node = gm.graph.get_attr(lifted_node)
/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/fx/graph.py:1801: UserWarning: Node bias target bias bias of does not reference an nn.Module, nn.Parameter, or buffer, which is what 'get_attr' Nodes typically target
warnings.warn(
/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/export/_unlift.py:75: UserWarning: Attempted to insert a get_attr Node with no underlying reference in the owning GraphModule! Call GraphModule.add_submodule to add the necessary submodule, GraphModule.add_parameter to add the necessary Parameter, or nn.Module.register_buffer to add the necessary buffer
getattr_node = gm.graph.get_attr(lifted_node)
/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/fx/graph.py:1801: UserWarning: Node bias target bias bias of does not reference an nn.Module, nn.Parameter, or buffer, which is what 'get_attr' Nodes typically target
warnings.warn(
71%|███████▏ | 60/84 [00:11<00:05, 4.27it/s]/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/export/_unlift.py:75: UserWarning: Attempted to insert a get_attr Node with no underlying reference in the owning GraphModule! Call GraphModule.add_submodule to add the necessary submodule, GraphModule.add_parameter to add the necessary Parameter, or nn.Module.register_buffer to add the necessary buffer
getattr_node = gm.graph.get_attr(lifted_node)
/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/fx/graph.py:1801: UserWarning: Node lifted_tensor_3 target lifted_tensor_3 lifted_tensor_3 of does not reference an nn.Module, nn.Parameter, or buffer, which is what 'get_attr' Nodes typically target
warnings.warn(
/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/export/_unlift.py:75: UserWarning: Attempted to insert a get_attr Node with no underlying reference in the owning GraphModule! Call GraphModule.add_submodule to add the necessary submodule, GraphModule.add_parameter to add the necessary Parameter, or nn.Module.register_buffer to add the necessary buffer
getattr_node = gm.graph.get_attr(lifted_node)
/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/fx/graph.py:1801: UserWarning: Node bias target bias bias of does not reference an nn.Module, nn.Parameter, or buffer, which is what 'get_attr' Nodes typically target
warnings.warn(
77%|███████▋ | 65/84 [00:12<00:02, 6.56it/s]/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/export/_unlift.py:75: UserWarning: Attempted to insert a get_attr Node with no underlying reference in the owning GraphModule! Call GraphModule.add_submodule to add the necessary submodule, GraphModule.add_parameter to add the necessary Parameter, or nn.Module.register_buffer to add the necessary buffer
getattr_node = gm.graph.get_attr(lifted_node)
/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/fx/graph.py:1801: UserWarning: Node bias target bias bias of does not reference an nn.Module, nn.Parameter, or buffer, which is what 'get_attr' Nodes typically target
warnings.warn(
79%|███████▊ | 66/84 [00:12<00:03, 5.80it/s]
81%|████████ | 68/84 [00:12<00:03, 5.30it/s]
82%|████████▏ | 69/84 [00:13<00:04, 3.16it/s]/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/export/_unlift.py:75: UserWarning: Attempted to insert a get_attr Node with no underlying reference in the owning GraphModule! Call GraphModule.add_submodule to add the necessary submodule, GraphModule.add_parameter to add the necessary Parameter, or nn.Module.register_buffer to add the necessary buffer
getattr_node = gm.graph.get_attr(lifted_node)
/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/fx/graph.py:1801: UserWarning: Node bias target bias bias of does not reference an nn.Module, nn.Parameter, or buffer, which is what 'get_attr' Nodes typically target
warnings.warn(
/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/export/_unlift.py:75: UserWarning: Attempted to insert a get_attr Node with no underlying reference in the owning GraphModule! Call GraphModule.add_submodule to add the necessary submodule, GraphModule.add_parameter to add the necessary Parameter, or nn.Module.register_buffer to add the necessary buffer
getattr_node = gm.graph.get_attr(lifted_node)
/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/fx/graph.py:1801: UserWarning: Node bias target bias bias of does not reference an nn.Module, nn.Parameter, or buffer, which is what 'get_attr' Nodes typically target
warnings.warn(
/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/export/_unlift.py:75: UserWarning: Attempted to insert a get_attr Node with no underlying reference in the owning GraphModule! Call GraphModule.add_submodule to add the necessary submodule, GraphModule.add_parameter to add the necessary Parameter, or nn.Module.register_buffer to add the necessary buffer
getattr_node = gm.graph.get_attr(lifted_node)
/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/fx/graph.py:1801: UserWarning: Node bias target bias bias of does not reference an nn.Module, nn.Parameter, or buffer, which is what 'get_attr' Nodes typically target
warnings.warn(
86%|████████▌ | 72/84 [00:14<00:02, 4.66it/s]/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/export/_unlift.py:75: UserWarning: Attempted to insert a get_attr Node with no underlying reference in the owning GraphModule! Call GraphModule.add_submodule to add the necessary submodule, GraphModule.add_parameter to add the necessary Parameter, or nn.Module.register_buffer to add the necessary buffer
getattr_node = gm.graph.get_attr(lifted_node)
/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/fx/graph.py:1801: UserWarning: Node bias target bias bias of does not reference an nn.Module, nn.Parameter, or buffer, which is what 'get_attr' Nodes typically target
warnings.warn(
/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/export/_unlift.py:75: UserWarning: Attempted to insert a get_attr Node with no underlying reference in the owning GraphModule! Call GraphModule.add_submodule to add the necessary submodule, GraphModule.add_parameter to add the necessary Parameter, or nn.Module.register_buffer to add the necessary buffer
getattr_node = gm.graph.get_attr(lifted_node)
/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/fx/graph.py:1801: UserWarning: Node bias target bias bias of does not reference an nn.Module, nn.Parameter, or buffer, which is what 'get_attr' Nodes typically target
warnings.warn(
/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/export/_unlift.py:75: UserWarning: Attempted to insert a get_attr Node with no underlying reference in the owning GraphModule! Call GraphModule.add_submodule to add the necessary submodule, GraphModule.add_parameter to add the necessary Parameter, or nn.Module.register_buffer to add the necessary buffer
getattr_node = gm.graph.get_attr(lifted_node)
/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/fx/graph.py:1801: UserWarning: Node bias target bias bias of does not reference an nn.Module, nn.Parameter, or buffer, which is what 'get_attr' Nodes typically target
warnings.warn(
88%|████████▊ | 74/84 [00:14<00:01, 5.58it/s]/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/export/_unlift.py:75: UserWarning: Attempted to insert a get_attr Node with no underlying reference in the owning GraphModule! Call GraphModule.add_submodule to add the necessary submodule, GraphModule.add_parameter to add the necessary Parameter, or nn.Module.register_buffer to add the necessary buffer
getattr_node = gm.graph.get_attr(lifted_node)
/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/fx/graph.py:1801: UserWarning: Node lifted_tensor_3 target lifted_tensor_3 lifted_tensor_3 of does not reference an nn.Module, nn.Parameter, or buffer, which is what 'get_attr' Nodes typically target
warnings.warn(
93%|█████████▎| 78/84 [00:14<00:00, 9.03it/s]/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/export/_unlift.py:75: UserWarning: Attempted to insert a get_attr Node with no underlying reference in the owning GraphModule! Call GraphModule.add_submodule to add the necessary submodule, GraphModule.add_parameter to add the necessary Parameter, or nn.Module.register_buffer to add the necessary buffer
getattr_node = gm.graph.get_attr(lifted_node)
/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/fx/graph.py:1801: UserWarning: Node bias target bias bias of does not reference an nn.Module, nn.Parameter, or buffer, which is what 'get_attr' Nodes typically target
warnings.warn(
/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/export/_unlift.py:75: UserWarning: Attempted to insert a get_attr Node with no underlying reference in the owning GraphModule! Call GraphModule.add_submodule to add the necessary submodule, GraphModule.add_parameter to add the necessary Parameter, or nn.Module.register_buffer to add the necessary buffer
getattr_node = gm.graph.get_attr(lifted_node)
/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/fx/graph.py:1801: UserWarning: Node bias target bias bias of does not reference an nn.Module, nn.Parameter, or buffer, which is what 'get_attr' Nodes typically target
warnings.warn(
95%|█████████▌| 80/84 [00:14<00:00, 6.90it/s]
98%|█████████▊| 82/84 [00:15<00:00, 4.98it/s]
100%|██████████| 84/84 [00:16<00:00, 4.54it/s]
100%|██████████| 84/84 [00:16<00:00, 5.20it/s]
The results
df = pandas.DataFrame(obs).sort_values(["dynamic", "name", "exporter"]).reset_index(drop=True)
df.to_csv("plot-exporter-coverage.csv", index=False)
df.to_excel("plot-exporter-coverage.xlsx")
for c in ["error", "error_step"]:
if c in df.columns:
df[c] = df[c].fillna("")
print(df)
abs rel success ... exporter error error_step
0 0.0 0.0 1 ... custom-nostrict
1 0.0 0.0 1 ... custom-nostrict-dec
2 0.0 0.0 1 ... custom-strict
3 0.0 0.0 1 ... custom-strict-dec
4 0.0 0.0 1 ... custom-tracing
.. ... ... ... ... ... ... ...
79 0.0 0.0 1 ... export-nostrict-dec
80 0.0 0.0 1 ... export-strict
81 0.0 0.0 1 ... export-strict-dec
82 0.0 0.0 1 ... export-tracing
83 NaN NaN 0 ... script number of input names provided (3) exceeded nu... export
[84 rows x 11 columns]
Errors if any or all successes.
piv = df.pivot(
index=["dynamic", "name"],
columns=["exporter"],
values="error_step" if "error_step" in df.columns else "success",
)
piv.to_excel("plot-exporter-coverage-summary.xlsx")
print(piv)
exporter custom-nostrict custom-nostrict-dec custom-strict ... export-strict-dec export-tracing script
dynamic name ...
0 AtenRollPos ...
AtenRollRelu ...
InplaceAdd ...
1 AtenRollPos ... export
AtenRollRelu ... export
InplaceAdd ... export
[6 rows x 14 columns]
Total running time of the script: (0 minutes 26.909 seconds)
Related examples
