Note
Go to the end to download the full example code.
Measures the exporter success on many test cases¶
All test cases can be found in module
experimental_experiment.torch_interpreter.eval.model_cases.
Page Supported Model Signatures shows the exported
program for many of those cases.
from experimental_experiment.args import get_parsed_args
script_args = get_parsed_args(
"plot_exporter_coverage",
description=__doc__,
exporter=("all", "an exporter to rerun"),
dynamic=("all", "use dyanmic shapes"),
case=(
"three",
"model cases, two for the first two (to test), "
"all to select all, a name or a regular expression fior a subset",
),
quiet=("1", "0 or 1"),
verbose=("1", "verbosity"),
expose="exporter,dyanmic,case,quiet,verbose",
)
exporters = (
(
"export-strict",
"export-strict-dec",
"export-nostrict",
"export-nostrict-dec",
"export-jit",
"export-tracing",
"custom-strict",
"custom-nostrict",
"custom-strict-dec",
"custom-nostrict-dec",
"custom-tracing",
"dynamo",
"dynamo-ir",
"script",
)
if script_args.exporter == "all"
else script_args.exporter.split(",")
)
dynamic = (0, 1) if script_args.dynamic == "all" else (int(script_args.dynamic),)
cases = None if script_args.case == "all" else script_args.case.split(",")
quiet = bool(int(script_args.quiet))
verbose = int(script_args.verbose)
import pandas
from experimental_experiment.torch_interpreter.eval import evaluation
obs = evaluation(
exporters=exporters, dynamic=dynamic, cases=cases, quiet=quiet, verbose=verbose
)
0%| | 0/84 [00:00<?, ?it/s]
2%|▏ | 2/84 [00:00<00:10, 7.78it/s]
5%|▍ | 4/84 [00:00<00:09, 8.79it/s]
6%|▌ | 5/84 [00:00<00:10, 7.68it/s]
10%|▉ | 8/84 [00:00<00:06, 11.05it/s]
12%|█▏ | 10/84 [00:01<00:11, 6.27it/s]
14%|█▍ | 12/84 [00:02<00:16, 4.40it/s]
15%|█▌ | 13/84 [00:03<00:27, 2.55it/s]
17%|█▋ | 14/84 [00:03<00:23, 3.02it/s]
18%|█▊ | 15/84 [00:03<00:19, 3.51it/s]
19%|█▉ | 16/84 [00:03<00:18, 3.58it/s]
21%|██▏ | 18/84 [00:03<00:14, 4.51it/s]
25%|██▌ | 21/84 [00:04<00:09, 6.93it/s]
27%|██▋ | 23/84 [00:04<00:09, 6.17it/s]
29%|██▊ | 24/84 [00:04<00:10, 5.49it/s]
31%|███ | 26/84 [00:05<00:17, 3.31it/s]
32%|███▏ | 27/84 [00:06<00:23, 2.47it/s]
35%|███▍ | 29/84 [00:06<00:15, 3.48it/s]
36%|███▌ | 30/84 [00:07<00:14, 3.64it/s]
38%|███▊ | 32/84 [00:07<00:11, 4.48it/s]
42%|████▏ | 35/84 [00:07<00:06, 7.04it/s]
44%|████▍ | 37/84 [00:07<00:07, 6.38it/s]
46%|████▋ | 39/84 [00:08<00:06, 6.69it/s]
48%|████▊ | 40/84 [00:09<00:13, 3.27it/s]
49%|████▉ | 41/84 [00:10<00:18, 2.30it/s]
51%|█████ | 43/84 [00:10<00:13, 3.07it/s]
52%|█████▏ | 44/84 [00:10<00:12, 3.26it/s]
55%|█████▍ | 46/84 [00:10<00:08, 4.30it/s]
58%|█████▊ | 49/84 [00:11<00:05, 6.58it/s]
61%|██████ | 51/84 [00:11<00:05, 5.91it/s]
62%|██████▏ | 52/84 [00:11<00:05, 5.72it/s]
64%|██████▍ | 54/84 [00:12<00:08, 3.39it/s]
65%|██████▌ | 55/84 [00:13<00:12, 2.25it/s]/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/export/_unlift.py:81: UserWarning: Attempted to insert a get_attr Node with no underlying reference in the owning GraphModule! Call GraphModule.add_submodule to add the necessary submodule, GraphModule.add_parameter to add the necessary Parameter, or nn.Module.register_buffer to add the necessary buffer
getattr_node = gm.graph.get_attr(lifted_node)
/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/fx/graph.py:1794: UserWarning: Node bias target bias bias of does not reference an nn.Module, nn.Parameter, or buffer, which is what 'get_attr' Nodes typically target
warnings.warn(
68%|██████▊ | 57/84 [00:13<00:08, 3.22it/s]/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/export/_unlift.py:81: UserWarning: Attempted to insert a get_attr Node with no underlying reference in the owning GraphModule! Call GraphModule.add_submodule to add the necessary submodule, GraphModule.add_parameter to add the necessary Parameter, or nn.Module.register_buffer to add the necessary buffer
getattr_node = gm.graph.get_attr(lifted_node)
/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/fx/graph.py:1794: UserWarning: Node bias target bias bias of does not reference an nn.Module, nn.Parameter, or buffer, which is what 'get_attr' Nodes typically target
warnings.warn(
/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/export/_unlift.py:81: UserWarning: Attempted to insert a get_attr Node with no underlying reference in the owning GraphModule! Call GraphModule.add_submodule to add the necessary submodule, GraphModule.add_parameter to add the necessary Parameter, or nn.Module.register_buffer to add the necessary buffer
getattr_node = gm.graph.get_attr(lifted_node)
/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/fx/graph.py:1794: UserWarning: Node bias target bias bias of does not reference an nn.Module, nn.Parameter, or buffer, which is what 'get_attr' Nodes typically target
warnings.warn(
69%|██████▉ | 58/84 [00:14<00:07, 3.52it/s]/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/export/_unlift.py:81: UserWarning: Attempted to insert a get_attr Node with no underlying reference in the owning GraphModule! Call GraphModule.add_submodule to add the necessary submodule, GraphModule.add_parameter to add the necessary Parameter, or nn.Module.register_buffer to add the necessary buffer
getattr_node = gm.graph.get_attr(lifted_node)
/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/fx/graph.py:1794: UserWarning: Node bias target bias bias of does not reference an nn.Module, nn.Parameter, or buffer, which is what 'get_attr' Nodes typically target
warnings.warn(
/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/export/_unlift.py:81: UserWarning: Attempted to insert a get_attr Node with no underlying reference in the owning GraphModule! Call GraphModule.add_submodule to add the necessary submodule, GraphModule.add_parameter to add the necessary Parameter, or nn.Module.register_buffer to add the necessary buffer
getattr_node = gm.graph.get_attr(lifted_node)
/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/fx/graph.py:1794: UserWarning: Node bias target bias bias of does not reference an nn.Module, nn.Parameter, or buffer, which is what 'get_attr' Nodes typically target
warnings.warn(
/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/export/_unlift.py:81: UserWarning: Attempted to insert a get_attr Node with no underlying reference in the owning GraphModule! Call GraphModule.add_submodule to add the necessary submodule, GraphModule.add_parameter to add the necessary Parameter, or nn.Module.register_buffer to add the necessary buffer
getattr_node = gm.graph.get_attr(lifted_node)
/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/fx/graph.py:1794: UserWarning: Node bias target bias bias of does not reference an nn.Module, nn.Parameter, or buffer, which is what 'get_attr' Nodes typically target
warnings.warn(
71%|███████▏ | 60/84 [00:14<00:05, 4.69it/s]/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/export/_unlift.py:81: UserWarning: Attempted to insert a get_attr Node with no underlying reference in the owning GraphModule! Call GraphModule.add_submodule to add the necessary submodule, GraphModule.add_parameter to add the necessary Parameter, or nn.Module.register_buffer to add the necessary buffer
getattr_node = gm.graph.get_attr(lifted_node)
/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/fx/graph.py:1794: UserWarning: Node lifted_tensor_3 target lifted_tensor_3 lifted_tensor_3 of does not reference an nn.Module, nn.Parameter, or buffer, which is what 'get_attr' Nodes typically target
warnings.warn(
76%|███████▌ | 64/84 [00:14<00:02, 7.94it/s]/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/export/_unlift.py:81: UserWarning: Attempted to insert a get_attr Node with no underlying reference in the owning GraphModule! Call GraphModule.add_submodule to add the necessary submodule, GraphModule.add_parameter to add the necessary Parameter, or nn.Module.register_buffer to add the necessary buffer
getattr_node = gm.graph.get_attr(lifted_node)
/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/fx/graph.py:1794: UserWarning: Node bias target bias bias of does not reference an nn.Module, nn.Parameter, or buffer, which is what 'get_attr' Nodes typically target
warnings.warn(
/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/export/_unlift.py:81: UserWarning: Attempted to insert a get_attr Node with no underlying reference in the owning GraphModule! Call GraphModule.add_submodule to add the necessary submodule, GraphModule.add_parameter to add the necessary Parameter, or nn.Module.register_buffer to add the necessary buffer
getattr_node = gm.graph.get_attr(lifted_node)
/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/fx/graph.py:1794: UserWarning: Node bias target bias bias of does not reference an nn.Module, nn.Parameter, or buffer, which is what 'get_attr' Nodes typically target
warnings.warn(
79%|███████▊ | 66/84 [00:15<00:03, 5.98it/s]
81%|████████ | 68/84 [00:16<00:05, 3.01it/s]
82%|████████▏ | 69/84 [00:17<00:05, 2.53it/s]/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/export/_unlift.py:81: UserWarning: Attempted to insert a get_attr Node with no underlying reference in the owning GraphModule! Call GraphModule.add_submodule to add the necessary submodule, GraphModule.add_parameter to add the necessary Parameter, or nn.Module.register_buffer to add the necessary buffer
getattr_node = gm.graph.get_attr(lifted_node)
/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/fx/graph.py:1794: UserWarning: Node bias target bias bias of does not reference an nn.Module, nn.Parameter, or buffer, which is what 'get_attr' Nodes typically target
warnings.warn(
85%|████████▍ | 71/84 [00:17<00:03, 3.37it/s]/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/export/_unlift.py:81: UserWarning: Attempted to insert a get_attr Node with no underlying reference in the owning GraphModule! Call GraphModule.add_submodule to add the necessary submodule, GraphModule.add_parameter to add the necessary Parameter, or nn.Module.register_buffer to add the necessary buffer
getattr_node = gm.graph.get_attr(lifted_node)
/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/fx/graph.py:1794: UserWarning: Node bias target bias bias of does not reference an nn.Module, nn.Parameter, or buffer, which is what 'get_attr' Nodes typically target
warnings.warn(
/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/export/_unlift.py:81: UserWarning: Attempted to insert a get_attr Node with no underlying reference in the owning GraphModule! Call GraphModule.add_submodule to add the necessary submodule, GraphModule.add_parameter to add the necessary Parameter, or nn.Module.register_buffer to add the necessary buffer
getattr_node = gm.graph.get_attr(lifted_node)
/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/fx/graph.py:1794: UserWarning: Node bias target bias bias of does not reference an nn.Module, nn.Parameter, or buffer, which is what 'get_attr' Nodes typically target
warnings.warn(
86%|████████▌ | 72/84 [00:17<00:03, 3.56it/s]/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/export/_unlift.py:81: UserWarning: Attempted to insert a get_attr Node with no underlying reference in the owning GraphModule! Call GraphModule.add_submodule to add the necessary submodule, GraphModule.add_parameter to add the necessary Parameter, or nn.Module.register_buffer to add the necessary buffer
getattr_node = gm.graph.get_attr(lifted_node)
/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/fx/graph.py:1794: UserWarning: Node bias target bias bias of does not reference an nn.Module, nn.Parameter, or buffer, which is what 'get_attr' Nodes typically target
warnings.warn(
/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/export/_unlift.py:81: UserWarning: Attempted to insert a get_attr Node with no underlying reference in the owning GraphModule! Call GraphModule.add_submodule to add the necessary submodule, GraphModule.add_parameter to add the necessary Parameter, or nn.Module.register_buffer to add the necessary buffer
getattr_node = gm.graph.get_attr(lifted_node)
/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/fx/graph.py:1794: UserWarning: Node bias target bias bias of does not reference an nn.Module, nn.Parameter, or buffer, which is what 'get_attr' Nodes typically target
warnings.warn(
/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/export/_unlift.py:81: UserWarning: Attempted to insert a get_attr Node with no underlying reference in the owning GraphModule! Call GraphModule.add_submodule to add the necessary submodule, GraphModule.add_parameter to add the necessary Parameter, or nn.Module.register_buffer to add the necessary buffer
getattr_node = gm.graph.get_attr(lifted_node)
/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/fx/graph.py:1794: UserWarning: Node bias target bias bias of does not reference an nn.Module, nn.Parameter, or buffer, which is what 'get_attr' Nodes typically target
warnings.warn(
88%|████████▊ | 74/84 [00:17<00:02, 4.46it/s]/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/export/_unlift.py:81: UserWarning: Attempted to insert a get_attr Node with no underlying reference in the owning GraphModule! Call GraphModule.add_submodule to add the necessary submodule, GraphModule.add_parameter to add the necessary Parameter, or nn.Module.register_buffer to add the necessary buffer
getattr_node = gm.graph.get_attr(lifted_node)
/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/fx/graph.py:1794: UserWarning: Node lifted_tensor_3 target lifted_tensor_3 lifted_tensor_3 of does not reference an nn.Module, nn.Parameter, or buffer, which is what 'get_attr' Nodes typically target
warnings.warn(
93%|█████████▎| 78/84 [00:18<00:00, 6.99it/s]/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/export/_unlift.py:81: UserWarning: Attempted to insert a get_attr Node with no underlying reference in the owning GraphModule! Call GraphModule.add_submodule to add the necessary submodule, GraphModule.add_parameter to add the necessary Parameter, or nn.Module.register_buffer to add the necessary buffer
getattr_node = gm.graph.get_attr(lifted_node)
/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/fx/graph.py:1794: UserWarning: Node bias target bias bias of does not reference an nn.Module, nn.Parameter, or buffer, which is what 'get_attr' Nodes typically target
warnings.warn(
94%|█████████▍| 79/84 [00:18<00:00, 5.66it/s]/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/export/_unlift.py:81: UserWarning: Attempted to insert a get_attr Node with no underlying reference in the owning GraphModule! Call GraphModule.add_submodule to add the necessary submodule, GraphModule.add_parameter to add the necessary Parameter, or nn.Module.register_buffer to add the necessary buffer
getattr_node = gm.graph.get_attr(lifted_node)
/home/xadupre/vv/this312/lib/python3.12/site-packages/torch/fx/graph.py:1794: UserWarning: Node bias target bias bias of does not reference an nn.Module, nn.Parameter, or buffer, which is what 'get_attr' Nodes typically target
warnings.warn(
95%|█████████▌| 80/84 [00:18<00:00, 4.84it/s]
98%|█████████▊| 82/84 [00:20<00:00, 2.86it/s]
99%|█████████▉| 83/84 [00:20<00:00, 2.35it/s]
100%|██████████| 84/84 [00:20<00:00, 4.03it/s]
The results
df = pandas.DataFrame(obs).sort_values(["dynamic", "name", "exporter"]).reset_index(drop=True)
df.to_csv("plot-exporter-coverage.csv", index=False)
df.to_excel("plot-exporter-coverage.xlsx")
for c in ["error", "error_step"]:
if c in df.columns:
df[c] = df[c].fillna("")
print(df)
abs rel dnan success model_cls ... name dynamic exporter error error_step
0 0.0 0.0 0.0 1 <class 'experimental_experiment.torch_interpre... ... AtenRollPos 0 custom-nostrict
1 0.0 0.0 0.0 1 <class 'experimental_experiment.torch_interpre... ... AtenRollPos 0 custom-nostrict-dec
2 0.0 0.0 0.0 1 <class 'experimental_experiment.torch_interpre... ... AtenRollPos 0 custom-strict
3 0.0 0.0 0.0 1 <class 'experimental_experiment.torch_interpre... ... AtenRollPos 0 custom-strict-dec
4 0.0 0.0 0.0 1 <class 'experimental_experiment.torch_interpre... ... AtenRollPos 0 custom-tracing
.. ... ... ... ... ... ... ... ... ... ... ...
79 0.0 0.0 0.0 1 <class 'experimental_experiment.torch_interpre... ... InplaceAdd 1 export-nostrict-dec
80 0.0 0.0 0.0 1 <class 'experimental_experiment.torch_interpre... ... InplaceAdd 1 export-strict
81 0.0 0.0 0.0 1 <class 'experimental_experiment.torch_interpre... ... InplaceAdd 1 export-strict-dec
82 0.0 0.0 0.0 1 <class 'experimental_experiment.torch_interpre... ... InplaceAdd 1 export-tracing
83 NaN NaN NaN 0 NaN ... InplaceAdd 1 script number of input names provided (3) exceeded nu... export
[84 rows x 12 columns]
Errors if any or all successes.
piv = df.pivot(
index=["dynamic", "name"],
columns=["exporter"],
values="error_step" if "error_step" in df.columns else "success",
)
piv.to_excel("plot-exporter-coverage-summary.xlsx")
print(piv)
exporter custom-nostrict custom-nostrict-dec custom-strict custom-strict-dec custom-tracing ... export-nostrict-dec export-strict export-strict-dec export-tracing script
dynamic name ...
0 AtenRollPos ...
AtenRollRelu ...
InplaceAdd ...
1 AtenRollPos ... export
AtenRollRelu ... export
InplaceAdd ... export
[6 rows x 14 columns]
Total running time of the script: (0 minutes 27.729 seconds)
Related examples
