Note

Go to the end to download the full example code.

Export with dynamic dimensions in `{0,1}`¶

The first version of torch.export.export() did not support any tensor with a dimension equal to 0, 1 if the dimension was expected to be dynamic. The latest versions offers more options. Let’s check it works. The experiments consists in exporting the model with different sets of inputs and checking the exported models works with all set of inputs.

Available input sets¶

import itertools
from tqdm import tqdm
import numpy as np
import pandas
import torch
from onnx_diagnostic import doc
from onnx_diagnostic.helpers import max_diff, string_type
from onnx_diagnostic.helpers.torch_helper import torch_deepcopy
from onnx_diagnostic.torch_models.hghub.model_inputs import get_untrained_model_with_inputs
from onnx_diagnostic.torch_export_patches.patch_inputs import use_dyn_not_str
from onnx_diagnostic.torch_export_patches import (
    torch_export_patches,
    register_additional_serialization_functions,
)


data = get_untrained_model_with_inputs("arnir0/Tiny-LLM", add_second_input=True)
model, dynamic_shapes = data["model"], data["dynamic_shapes"]

The trained model can be obtained with:

MODEL_NAME = "arnir0/Tiny-LLM"
tokenizer = transformers.AutoTokenizer.from_pretrained(MODEL_NAME)
model = transformers.AutoModelForCausalLM.from_pretrained(MODEL_NAME)

input_sets = {k: v for k, v in data.items() if k.startswith("inputs")}

for k, v in input_sets.items():
    print(f"{k:20}: {string_type(v, with_shape=True)}")

inputs              : dict(input_ids:T7s2x3,attention_mask:T7s2x33,position_ids:T7s2x3,past_key_values:DynamicCache(key_cache=#1[T1s2x1x30x96], value_cache=#1[T1s2x1x30x96]))
inputs_prompt       : dict(input_ids:T7s1x11)
inputs2             : dict(input_ids:T7s3x4,attention_mask:T7s3x35,position_ids:T7s3x4,past_key_values:DynamicCache(key_cache=#1[T1s3x1x31x96], value_cache=#1[T1s3x1x31x96]))
inputs_empty_cache  : dict(input_ids:T7s2x3,attention_mask:T7s2x3,position_ids:T7s2x3,past_key_values:DynamicCache(key_cache=#1[T1s2x1x0x96], value_cache=#1[T1s2x1x0x96]))
inputs_batch1       : dict(input_ids:T7s1x3,attention_mask:T7s1x33,position_ids:T7s1x3,past_key_values:DynamicCache(key_cache=#1[T1s1x1x30x96], value_cache=#1[T1s1x1x30x96]))

The dynamic shapes are:

print(f"dynamic_shapes: {string_type(dynamic_shapes)}")

dynamic_shapes: dict(input_ids:{0:DYN(batch),1:DYN(seq_length)},attention_mask:{0:DYN(batch),1:DYN(cache+seq)},position_ids:{0:DYN(batch),1:DYN(seq_length)},past_key_values:#2[{0:DYN(batch),2:DYN(cache_length)},{0:DYN(batch),2:DYN(cache_length)}])

dynamic_shapes = use_dyn_not_str(dynamic_shapes)
print(f"dynamic_shapes: {string_type(dynamic_shapes)}")

dynamic_shapes: dict(input_ids:{0:DYNAMIC,1:DYNAMIC},attention_mask:{0:DYNAMIC,1:DYNAMIC},position_ids:{0:DYNAMIC,1:DYNAMIC},past_key_values:#2[{0:DYNAMIC,2:DYNAMIC},{0:DYNAMIC,2:DYNAMIC}])

Let’s check they all work and compute the expected values. We use deepcopy because caches are usually modified inplace.

expected = {}
for k, v in input_sets.items():
    expected[k] = model(**torch_deepcopy(v))
    print(f"{k:20}: {string_type(expected[k], with_shape=True)}")

inputs              : CausalLMOutputWithPast(logits:T1s2x3x32000,past_key_values:DynamicCache(key_cache=#1[T1s2x1x33x96], value_cache=#1[T1s2x1x33x96]))
inputs_prompt       : CausalLMOutputWithPast(logits:T1s1x11x32000,past_key_values:DynamicCache(key_cache=#1[T1s1x1x11x96], value_cache=#1[T1s1x1x11x96]))
inputs2             : CausalLMOutputWithPast(logits:T1s3x4x32000,past_key_values:DynamicCache(key_cache=#1[T1s3x1x35x96], value_cache=#1[T1s3x1x35x96]))
inputs_empty_cache  : CausalLMOutputWithPast(logits:T1s2x3x32000,past_key_values:DynamicCache(key_cache=#1[T1s2x1x3x96], value_cache=#1[T1s2x1x3x96]))
inputs_batch1       : CausalLMOutputWithPast(logits:T1s1x3x32000,past_key_values:DynamicCache(key_cache=#1[T1s1x1x33x96], value_cache=#1[T1s1x1x33x96]))

Export with options¶

We try to export with the following options:

cache registration: register cache serialization with onnx_diagnostic.torch_export_patches.register_additional_serialization_functions()
oblivious: an option to remove some the exception raises by the exporter
rt: see prefer_deferred_runtime_asserts_over_guards in torch.export.export()
cache_patch: patches the model before exporting with onnx_diagnostic.torch_export_patches.torch_export_patches()

Some function first.

def export_model(
    model,
    dynamic_shapes,
    inputs,
    cache=False,
    oblivious=False,
    rt=False,
    cache_patch=False,
    strict=False,
):
    if cache and not cache_patch:
        with register_additional_serialization_functions(patch_transformers=True):
            return export_model(
                model, dynamic_shapes, inputs, oblivious=oblivious, rt=rt, strict=strict
            )
    if cache_patch:
        with torch_export_patches(
            patch_torch=cache_patch in ("all", "torch", True, 1),
            patch_transformers=cache_patch in ("all", "transformers", True, 1),
        ):
            return export_model(
                model, dynamic_shapes, inputs, oblivious=oblivious, rt=rt, strict=strict
            )
    if oblivious:
        with torch.fx.experimental._config.patch(backed_size_oblivious=True):
            return export_model(model, dynamic_shapes, inputs, rt=rt, strict=strict)
    return torch.export.export(
        model,
        (),
        inputs,
        dynamic_shapes=dynamic_shapes,
        strict=strict,
        prefer_deferred_runtime_asserts_over_guards=rt,
    )


def try_export_model(
    model,
    dynamic_shapes,
    inputs,
    cache=False,
    oblivious=False,
    rt=False,
    cache_patch=False,
    strict=False,
):
    try:
        return export_model(
            model,
            dynamic_shapes,
            inputs,
            cache=cache,
            oblivious=oblivious,
            rt=rt,
            cache_patch=cache_patch,
            strict=strict,
        )
    except Exception as e:
        return e


def validation(ep, input_sets, expected):
    mod = ep.module()
    for k, v in input_sets.items():
        try:
            got = mod(**torch_deepcopy(v))
        except Exception as e:
            yield k, e
            continue
        yield k, max_diff(expected[k], got, verbose=0)

The main loop¶

results = []

possibilities = [*[[0, 1] for _ in range(5)], list(input_sets)]
possibilities[1] = [0, "all", "torch", "transformers"]
with tqdm(list(itertools.product(*possibilities))) as pbar:
    for cache, cache_patch, strict, oblivious, rt, inputs in pbar:
        if cache_patch and not cache:
            # patches include caches.
            continue
        kwargs = dict(
            cache=cache, cache_patch=cache_patch, strict=strict, oblivious=oblivious, rt=rt
        )
        legend = "-".join(
            (k if isinstance(v, int) else f"{k}:{v}") for k, v in kwargs.items() if v
        )
        legend = f"{legend}/{inputs}"
        pbar.set_description(f"{legend} EXPORT")

        # export
        ep = try_export_model(
            model, dynamic_shapes, torch_deepcopy(input_sets[inputs]), **kwargs
        )
        if isinstance(ep, Exception):
            obs = {
                **kwargs,
                "export_with": inputs,
                "EXPORT": 0,
                "ERR-EXPORT": str(ep).split("\n")[0],
            }
            results.append(obs)
            continue

        pbar.set_description(f"{legend} VALIDATE")
        common = {**kwargs, "export_with": inputs, "EXPORT": 1}
        for inp, res in validation(ep, input_sets, expected):
            if isinstance(res, Exception):
                obs = {
                    **common,
                    "run_with": inp,
                    "ERR-RUN": str(res).split("\n")[0],
                    "WORKS": 0,
                }
            else:
                obs = {
                    **common,
                    "run_with": inp,
                    "WORKS": int(~np.isnan(res["abs"]) and res["abs"] < 1e-3),
                }
            results.append(obs)

  0%|          | 0/320 [00:00<?, ?it/s]
/inputs EXPORT:   0%|          | 0/320 [00:00<?, ?it/s]
/inputs VALIDATE:   0%|          | 0/320 [00:00<?, ?it/s]
/inputs VALIDATE:   0%|          | 1/320 [00:00<04:56,  1.08it/s]
/inputs_prompt EXPORT:   0%|          | 1/320 [00:00<04:56,  1.08it/s]
/inputs2 EXPORT:   0%|          | 1/320 [00:00<04:56,  1.08it/s]
/inputs2 VALIDATE:   0%|          | 1/320 [00:01<04:56,  1.08it/s]
/inputs2 VALIDATE:   1%|          | 3/320 [00:01<02:59,  1.76it/s]
/inputs_empty_cache EXPORT:   1%|          | 3/320 [00:01<02:59,  1.76it/s]
/inputs_empty_cache EXPORT:   1%|▏         | 4/320 [00:02<03:19,  1.59it/s]
/inputs_batch1 EXPORT:   1%|▏         | 4/320 [00:02<03:19,  1.59it/s]
/inputs_batch1 EXPORT:   2%|▏         | 5/320 [00:03<03:19,  1.58it/s]
rt/inputs EXPORT:   2%|▏         | 5/320 [00:03<03:19,  1.58it/s]
rt/inputs VALIDATE:   2%|▏         | 5/320 [00:04<03:19,  1.58it/s]
rt/inputs VALIDATE:   2%|▏         | 6/320 [00:04<03:48,  1.38it/s]
rt/inputs_prompt EXPORT:   2%|▏         | 6/320 [00:04<03:48,  1.38it/s]
rt/inputs2 EXPORT:   2%|▏         | 6/320 [00:04<03:48,  1.38it/s]
rt/inputs2 VALIDATE:   2%|▏         | 6/320 [00:04<03:48,  1.38it/s]
rt/inputs2 VALIDATE:   2%|▎         | 8/320 [00:04<03:02,  1.71it/s]
rt/inputs_empty_cache EXPORT:   2%|▎         | 8/320 [00:04<03:02,  1.71it/s]
rt/inputs_empty_cache EXPORT:   3%|▎         | 9/320 [00:05<03:16,  1.59it/s]
rt/inputs_batch1 EXPORT:   3%|▎         | 9/320 [00:05<03:16,  1.59it/s]
rt/inputs_batch1 EXPORT:   3%|▎         | 10/320 [00:06<03:13,  1.61it/s]
oblivious/inputs EXPORT:   3%|▎         | 10/320 [00:06<03:13,  1.61it/s]
oblivious/inputs VALIDATE:   3%|▎         | 10/320 [00:07<03:13,  1.61it/s]
oblivious/inputs VALIDATE:   3%|▎         | 11/320 [00:07<03:36,  1.42it/s]
oblivious/inputs_prompt EXPORT:   3%|▎         | 11/320 [00:07<03:36,  1.42it/s]
oblivious/inputs2 EXPORT:   3%|▎         | 11/320 [00:07<03:36,  1.42it/s]
oblivious/inputs2 VALIDATE:   3%|▎         | 11/320 [00:08<03:36,  1.42it/s]
oblivious/inputs2 VALIDATE:   4%|▍         | 13/320 [00:08<03:19,  1.54it/s]
oblivious/inputs_empty_cache EXPORT:   4%|▍         | 13/320 [00:08<03:19,  1.54it/s]
oblivious/inputs_empty_cache VALIDATE:   4%|▍         | 13/320 [00:09<03:19,  1.54it/s]
oblivious/inputs_empty_cache VALIDATE:   4%|▍         | 14/320 [00:09<03:30,  1.45it/s]
oblivious/inputs_batch1 EXPORT:   4%|▍         | 14/320 [00:09<03:30,  1.45it/s]
oblivious/inputs_batch1 VALIDATE:   4%|▍         | 14/320 [00:10<03:30,  1.45it/s]
oblivious/inputs_batch1 VALIDATE:   5%|▍         | 15/320 [00:10<03:39,  1.39it/s]
oblivious-rt/inputs EXPORT:   5%|▍         | 15/320 [00:10<03:39,  1.39it/s]
oblivious-rt/inputs VALIDATE:   5%|▍         | 15/320 [00:10<03:39,  1.39it/s]
oblivious-rt/inputs VALIDATE:   5%|▌         | 16/320 [00:10<03:52,  1.31it/s]
oblivious-rt/inputs_prompt EXPORT:   5%|▌         | 16/320 [00:10<03:52,  1.31it/s]
oblivious-rt/inputs2 EXPORT:   5%|▌         | 16/320 [00:10<03:52,  1.31it/s]
oblivious-rt/inputs2 VALIDATE:   5%|▌         | 16/320 [00:11<03:52,  1.31it/s]
oblivious-rt/inputs2 VALIDATE:   6%|▌         | 18/320 [00:11<03:02,  1.66it/s]
oblivious-rt/inputs_empty_cache EXPORT:   6%|▌         | 18/320 [00:11<03:02,  1.66it/s]
oblivious-rt/inputs_empty_cache VALIDATE:   6%|▌         | 18/320 [00:12<03:02,  1.66it/s]
oblivious-rt/inputs_empty_cache VALIDATE:   6%|▌         | 19/320 [00:12<03:14,  1.55it/s]
oblivious-rt/inputs_batch1 EXPORT:   6%|▌         | 19/320 [00:12<03:14,  1.55it/s]
oblivious-rt/inputs_batch1 VALIDATE:   6%|▌         | 19/320 [00:13<03:14,  1.55it/s]
oblivious-rt/inputs_batch1 VALIDATE:   6%|▋         | 20/320 [00:13<03:21,  1.49it/s]
strict/inputs EXPORT:   6%|▋         | 20/320 [00:13<03:21,  1.49it/s]
strict/inputs EXPORT:   7%|▋         | 21/320 [00:17<07:35,  1.52s/it]
strict/inputs_prompt EXPORT:   7%|▋         | 21/320 [00:17<07:35,  1.52s/it]
strict/inputs2 EXPORT:   7%|▋         | 21/320 [00:17<07:35,  1.52s/it]
strict/inputs2 EXPORT:   7%|▋         | 23/320 [00:18<05:40,  1.15s/it]
strict/inputs_empty_cache EXPORT:   7%|▋         | 23/320 [00:18<05:40,  1.15s/it]
strict/inputs_empty_cache EXPORT:   8%|▊         | 24/320 [00:19<05:43,  1.16s/it]
strict/inputs_batch1 EXPORT:   8%|▊         | 24/320 [00:19<05:43,  1.16s/it]
strict/inputs_batch1 EXPORT:   8%|▊         | 25/320 [00:21<06:30,  1.32s/it]
strict-rt/inputs EXPORT:   8%|▊         | 25/320 [00:21<06:30,  1.32s/it]
strict-rt/inputs EXPORT:   8%|▊         | 26/320 [00:23<07:17,  1.49s/it]
strict-rt/inputs_prompt EXPORT:   8%|▊         | 26/320 [00:23<07:17,  1.49s/it]
strict-rt/inputs2 EXPORT:   8%|▊         | 26/320 [00:23<07:17,  1.49s/it]
strict-rt/inputs2 EXPORT:   9%|▉         | 28/320 [00:25<06:14,  1.28s/it]
strict-rt/inputs_empty_cache EXPORT:   9%|▉         | 28/320 [00:25<06:14,  1.28s/it]
strict-rt/inputs_empty_cache EXPORT:   9%|▉         | 29/320 [00:27<07:31,  1.55s/it]
strict-rt/inputs_batch1 EXPORT:   9%|▉         | 29/320 [00:27<07:31,  1.55s/it]
strict-rt/inputs_batch1 EXPORT:   9%|▉         | 30/320 [00:29<07:42,  1.59s/it]
strict-oblivious/inputs EXPORT:   9%|▉         | 30/320 [00:29<07:42,  1.59s/it]
strict-oblivious/inputs EXPORT:  10%|▉         | 31/320 [00:30<07:11,  1.49s/it]
strict-oblivious/inputs_prompt EXPORT:  10%|▉         | 31/320 [00:30<07:11,  1.49s/it]
strict-oblivious/inputs2 EXPORT:  10%|▉         | 31/320 [00:30<07:11,  1.49s/it]
strict-oblivious/inputs2 EXPORT:  10%|█         | 33/320 [00:32<05:20,  1.12s/it]
strict-oblivious/inputs_empty_cache EXPORT:  10%|█         | 33/320 [00:32<05:20,  1.12s/it]
strict-oblivious/inputs_empty_cache EXPORT:  11%|█         | 34/320 [00:33<05:19,  1.12s/it]
strict-oblivious/inputs_batch1 EXPORT:  11%|█         | 34/320 [00:33<05:19,  1.12s/it]
strict-oblivious/inputs_batch1 EXPORT:  11%|█         | 35/320 [00:35<06:29,  1.37s/it]
strict-oblivious-rt/inputs EXPORT:  11%|█         | 35/320 [00:35<06:29,  1.37s/it]
strict-oblivious-rt/inputs EXPORT:  11%|█▏        | 36/320 [00:37<07:18,  1.54s/it]
strict-oblivious-rt/inputs_prompt EXPORT:  11%|█▏        | 36/320 [00:37<07:18,  1.54s/it]
strict-oblivious-rt/inputs2 EXPORT:  11%|█▏        | 36/320 [00:37<07:18,  1.54s/it]
strict-oblivious-rt/inputs2 EXPORT:  12%|█▏        | 38/320 [00:39<06:48,  1.45s/it]
strict-oblivious-rt/inputs_empty_cache EXPORT:  12%|█▏        | 38/320 [00:39<06:48,  1.45s/it]
strict-oblivious-rt/inputs_empty_cache EXPORT:  12%|█▏        | 39/320 [00:41<07:17,  1.56s/it]
strict-oblivious-rt/inputs_batch1 EXPORT:  12%|█▏        | 39/320 [00:41<07:17,  1.56s/it]
strict-oblivious-rt/inputs_batch1 EXPORT:  12%|█▎        | 40/320 [00:43<07:39,  1.64s/it]
cache/inputs EXPORT:  12%|█▎        | 40/320 [00:43<07:39,  1.64s/it]
cache/inputs VALIDATE:  12%|█▎        | 40/320 [00:44<07:39,  1.64s/it]
cache/inputs VALIDATE:  50%|█████     | 161/320 [00:44<00:07, 22.52it/s]
cache/inputs_prompt EXPORT:  50%|█████     | 161/320 [00:44<00:07, 22.52it/s]
cache/inputs2 EXPORT:  50%|█████     | 161/320 [00:44<00:07, 22.52it/s]
cache/inputs2 VALIDATE:  50%|█████     | 161/320 [00:45<00:07, 22.52it/s]
cache/inputs_empty_cache EXPORT:  50%|█████     | 161/320 [00:45<00:07, 22.52it/s]
cache/inputs_batch1 EXPORT:  50%|█████     | 161/320 [00:46<00:07, 22.52it/s]
cache/inputs_batch1 EXPORT:  52%|█████▏    | 165/320 [00:46<00:10, 15.21it/s]
cache-rt/inputs EXPORT:  52%|█████▏    | 165/320 [00:46<00:10, 15.21it/s]
cache-rt/inputs VALIDATE:  52%|█████▏    | 165/320 [00:47<00:10, 15.21it/s]
cache-rt/inputs_prompt EXPORT:  52%|█████▏    | 165/320 [00:47<00:10, 15.21it/s]
cache-rt/inputs2 EXPORT:  52%|█████▏    | 165/320 [00:47<00:10, 15.21it/s]
cache-rt/inputs2 VALIDATE:  52%|█████▏    | 165/320 [00:48<00:10, 15.21it/s]
cache-rt/inputs2 VALIDATE:  52%|█████▎    | 168/320 [00:48<00:13, 11.24it/s]
cache-rt/inputs_empty_cache EXPORT:  52%|█████▎    | 168/320 [00:48<00:13, 11.24it/s]
cache-rt/inputs_batch1 EXPORT:  52%|█████▎    | 168/320 [00:49<00:13, 11.24it/s]
cache-oblivious/inputs EXPORT:  52%|█████▎    | 168/320 [00:49<00:13, 11.24it/s]
cache-oblivious/inputs VALIDATE:  52%|█████▎    | 168/320 [00:50<00:13, 11.24it/s]
cache-oblivious/inputs VALIDATE:  53%|█████▎    | 171/320 [00:50<00:18,  7.97it/s]
cache-oblivious/inputs_prompt EXPORT:  53%|█████▎    | 171/320 [00:50<00:18,  7.97it/s]
cache-oblivious/inputs2 EXPORT:  53%|█████▎    | 171/320 [00:50<00:18,  7.97it/s]
cache-oblivious/inputs2 VALIDATE:  53%|█████▎    | 171/320 [00:52<00:18,  7.97it/s]
cache-oblivious/inputs2 VALIDATE:  54%|█████▍    | 173/320 [00:52<00:23,  6.25it/s]
cache-oblivious/inputs_empty_cache EXPORT:  54%|█████▍    | 173/320 [00:52<00:23,  6.25it/s]
cache-oblivious/inputs_empty_cache VALIDATE:  54%|█████▍    | 173/320 [00:52<00:23,  6.25it/s]
cache-oblivious/inputs_batch1 EXPORT:  54%|█████▍    | 173/320 [00:52<00:23,  6.25it/s]
cache-oblivious/inputs_batch1 VALIDATE:  54%|█████▍    | 173/320 [00:53<00:23,  6.25it/s]
cache-oblivious/inputs_batch1 VALIDATE:  55%|█████▍    | 175/320 [00:53<00:29,  4.93it/s]
cache-oblivious-rt/inputs EXPORT:  55%|█████▍    | 175/320 [00:53<00:29,  4.93it/s]
cache-oblivious-rt/inputs VALIDATE:  55%|█████▍    | 175/320 [00:54<00:29,  4.93it/s]
cache-oblivious-rt/inputs VALIDATE:  55%|█████▌    | 176/320 [00:54<00:33,  4.31it/s]
cache-oblivious-rt/inputs_prompt EXPORT:  55%|█████▌    | 176/320 [00:54<00:33,  4.31it/s]
cache-oblivious-rt/inputs2 EXPORT:  55%|█████▌    | 176/320 [00:54<00:33,  4.31it/s]
cache-oblivious-rt/inputs2 VALIDATE:  55%|█████▌    | 176/320 [00:55<00:33,  4.31it/s]
cache-oblivious-rt/inputs2 VALIDATE:  56%|█████▌    | 178/320 [00:55<00:35,  4.00it/s]
cache-oblivious-rt/inputs_empty_cache EXPORT:  56%|█████▌    | 178/320 [00:55<00:35,  4.00it/s]
cache-oblivious-rt/inputs_empty_cache VALIDATE:  56%|█████▌    | 178/320 [00:55<00:35,  4.00it/s]
cache-oblivious-rt/inputs_empty_cache VALIDATE:  56%|█████▌    | 179/320 [00:55<00:41,  3.37it/s]
cache-oblivious-rt/inputs_batch1 EXPORT:  56%|█████▌    | 179/320 [00:55<00:41,  3.37it/s]
cache-oblivious-rt/inputs_batch1 VALIDATE:  56%|█████▌    | 179/320 [00:56<00:41,  3.37it/s]
cache-oblivious-rt/inputs_batch1 VALIDATE:  56%|█████▋    | 180/320 [00:56<00:48,  2.91it/s]
cache-strict/inputs EXPORT:  56%|█████▋    | 180/320 [00:56<00:48,  2.91it/s]
cache-strict/inputs EXPORT:  57%|█████▋    | 181/320 [00:57<01:03,  2.18it/s]
cache-strict/inputs_prompt EXPORT:  57%|█████▋    | 181/320 [00:57<01:03,  2.18it/s]
cache-strict/inputs2 EXPORT:  57%|█████▋    | 181/320 [00:57<01:03,  2.18it/s]
cache-strict/inputs2 EXPORT:  57%|█████▋    | 183/320 [00:59<01:07,  2.04it/s]
cache-strict/inputs_empty_cache EXPORT:  57%|█████▋    | 183/320 [00:59<01:07,  2.04it/s]
cache-strict/inputs_empty_cache EXPORT:  57%|█████▊    | 184/320 [01:00<01:19,  1.70it/s]
cache-strict/inputs_batch1 EXPORT:  57%|█████▊    | 184/320 [01:00<01:19,  1.70it/s]
cache-strict/inputs_batch1 EXPORT:  58%|█████▊    | 185/320 [01:01<01:48,  1.25it/s]
cache-strict-rt/inputs EXPORT:  58%|█████▊    | 185/320 [01:01<01:48,  1.25it/s]
cache-strict-rt/inputs EXPORT:  58%|█████▊    | 186/320 [01:04<02:23,  1.07s/it]
cache-strict-rt/inputs_prompt EXPORT:  58%|█████▊    | 186/320 [01:04<02:23,  1.07s/it]
cache-strict-rt/inputs2 EXPORT:  58%|█████▊    | 186/320 [01:04<02:23,  1.07s/it]
cache-strict-rt/inputs2 EXPORT:  59%|█████▉    | 188/320 [01:07<02:46,  1.26s/it]
cache-strict-rt/inputs_empty_cache EXPORT:  59%|█████▉    | 188/320 [01:07<02:46,  1.26s/it]
cache-strict-rt/inputs_empty_cache EXPORT:  59%|█████▉    | 189/320 [01:09<03:14,  1.48s/it]
cache-strict-rt/inputs_batch1 EXPORT:  59%|█████▉    | 189/320 [01:09<03:14,  1.48s/it]
cache-strict-rt/inputs_batch1 EXPORT:  59%|█████▉    | 190/320 [01:11<03:21,  1.55s/it]
cache-strict-oblivious/inputs EXPORT:  59%|█████▉    | 190/320 [01:11<03:21,  1.55s/it]
cache-strict-oblivious/inputs EXPORT:  60%|█████▉    | 191/320 [01:12<03:07,  1.45s/it]
cache-strict-oblivious/inputs_prompt EXPORT:  60%|█████▉    | 191/320 [01:12<03:07,  1.45s/it]
cache-strict-oblivious/inputs2 EXPORT:  60%|█████▉    | 191/320 [01:12<03:07,  1.45s/it]
cache-strict-oblivious/inputs2 EXPORT:  60%|██████    | 193/320 [01:13<02:14,  1.06s/it]
cache-strict-oblivious/inputs_empty_cache EXPORT:  60%|██████    | 193/320 [01:13<02:14,  1.06s/it]
cache-strict-oblivious/inputs_empty_cache EXPORT:  61%|██████    | 194/320 [01:14<02:15,  1.07s/it]
cache-strict-oblivious/inputs_batch1 EXPORT:  61%|██████    | 194/320 [01:14<02:15,  1.07s/it]
cache-strict-oblivious/inputs_batch1 EXPORT:  61%|██████    | 195/320 [01:16<02:38,  1.27s/it]
cache-strict-oblivious-rt/inputs EXPORT:  61%|██████    | 195/320 [01:16<02:38,  1.27s/it]
cache-strict-oblivious-rt/inputs EXPORT:  61%|██████▏   | 196/320 [01:18<03:00,  1.46s/it]
cache-strict-oblivious-rt/inputs_prompt EXPORT:  61%|██████▏   | 196/320 [01:18<03:00,  1.46s/it]
cache-strict-oblivious-rt/inputs2 EXPORT:  61%|██████▏   | 196/320 [01:18<03:00,  1.46s/it]
cache-strict-oblivious-rt/inputs2 EXPORT:  62%|██████▏   | 198/320 [01:21<03:03,  1.50s/it]
cache-strict-oblivious-rt/inputs_empty_cache EXPORT:  62%|██████▏   | 198/320 [01:21<03:03,  1.50s/it]
cache-strict-oblivious-rt/inputs_empty_cache EXPORT:  62%|██████▏   | 199/320 [01:23<03:15,  1.62s/it]
cache-strict-oblivious-rt/inputs_batch1 EXPORT:  62%|██████▏   | 199/320 [01:23<03:15,  1.62s/it]
cache-strict-oblivious-rt/inputs_batch1 EXPORT:  62%|██████▎   | 200/320 [01:25<03:22,  1.69s/it]
cache-cache_patch:all/inputs EXPORT:  62%|██████▎   | 200/320 [01:25<03:22,  1.69s/it]
cache-cache_patch:all/inputs VALIDATE:  62%|██████▎   | 200/320 [01:26<03:22,  1.69s/it]
cache-cache_patch:all/inputs VALIDATE:  63%|██████▎   | 201/320 [01:26<02:57,  1.49s/it]
cache-cache_patch:all/inputs_prompt EXPORT:  63%|██████▎   | 201/320 [01:26<02:57,  1.49s/it]
cache-cache_patch:all/inputs2 EXPORT:  63%|██████▎   | 201/320 [01:26<02:57,  1.49s/it]
cache-cache_patch:all/inputs2 VALIDATE:  63%|██████▎   | 201/320 [01:27<02:57,  1.49s/it]
cache-cache_patch:all/inputs2 VALIDATE:  63%|██████▎   | 203/320 [01:27<01:59,  1.02s/it]
cache-cache_patch:all/inputs_empty_cache EXPORT:  63%|██████▎   | 203/320 [01:27<01:59,  1.02s/it]
cache-cache_patch:all/inputs_empty_cache VALIDATE:  63%|██████▎   | 203/320 [01:27<01:59,  1.02s/it]
cache-cache_patch:all/inputs_empty_cache VALIDATE:  64%|██████▍   | 204/320 [01:27<01:51,  1.04it/s]
cache-cache_patch:all/inputs_batch1 EXPORT:  64%|██████▍   | 204/320 [01:27<01:51,  1.04it/s]
cache-cache_patch:all/inputs_batch1 VALIDATE:  64%|██████▍   | 204/320 [01:28<01:51,  1.04it/s]
cache-cache_patch:all/inputs_batch1 VALIDATE:  64%|██████▍   | 205/320 [01:28<01:40,  1.15it/s]
cache-cache_patch:all-rt/inputs EXPORT:  64%|██████▍   | 205/320 [01:28<01:40,  1.15it/s]
cache-cache_patch:all-rt/inputs VALIDATE:  64%|██████▍   | 205/320 [01:29<01:40,  1.15it/s]
cache-cache_patch:all-rt/inputs VALIDATE:  64%|██████▍   | 206/320 [01:29<01:37,  1.17it/s]
cache-cache_patch:all-rt/inputs_prompt EXPORT:  64%|██████▍   | 206/320 [01:29<01:37,  1.17it/s]
cache-cache_patch:all-rt/inputs2 EXPORT:  64%|██████▍   | 206/320 [01:29<01:37,  1.17it/s]
cache-cache_patch:all-rt/inputs2 VALIDATE:  64%|██████▍   | 206/320 [01:30<01:37,  1.17it/s]
cache-cache_patch:all-rt/inputs2 VALIDATE:  65%|██████▌   | 208/320 [01:30<01:13,  1.52it/s]
cache-cache_patch:all-rt/inputs_empty_cache EXPORT:  65%|██████▌   | 208/320 [01:30<01:13,  1.52it/s]
cache-cache_patch:all-rt/inputs_empty_cache VALIDATE:  65%|██████▌   | 208/320 [01:30<01:13,  1.52it/s]
cache-cache_patch:all-rt/inputs_empty_cache VALIDATE:  65%|██████▌   | 209/320 [01:30<01:15,  1.47it/s]
cache-cache_patch:all-rt/inputs_batch1 EXPORT:  65%|██████▌   | 209/320 [01:30<01:15,  1.47it/s]
cache-cache_patch:all-rt/inputs_batch1 VALIDATE:  65%|██████▌   | 209/320 [01:31<01:15,  1.47it/s]
cache-cache_patch:all-rt/inputs_batch1 VALIDATE:  66%|██████▌   | 210/320 [01:31<01:13,  1.51it/s]
cache-cache_patch:all-oblivious/inputs EXPORT:  66%|██████▌   | 210/320 [01:31<01:13,  1.51it/s]
cache-cache_patch:all-oblivious/inputs VALIDATE:  66%|██████▌   | 210/320 [01:32<01:13,  1.51it/s]
cache-cache_patch:all-oblivious/inputs VALIDATE:  66%|██████▌   | 211/320 [01:32<01:25,  1.28it/s]
cache-cache_patch:all-oblivious/inputs_prompt EXPORT:  66%|██████▌   | 211/320 [01:32<01:25,  1.28it/s]
cache-cache_patch:all-oblivious/inputs2 EXPORT:  66%|██████▌   | 211/320 [01:32<01:25,  1.28it/s]
cache-cache_patch:all-oblivious/inputs2 VALIDATE:  66%|██████▌   | 211/320 [01:33<01:25,  1.28it/s]
cache-cache_patch:all-oblivious/inputs2 VALIDATE:  67%|██████▋   | 213/320 [01:33<01:10,  1.52it/s]
cache-cache_patch:all-oblivious/inputs_empty_cache EXPORT:  67%|██████▋   | 213/320 [01:33<01:10,  1.52it/s]
cache-cache_patch:all-oblivious/inputs_empty_cache VALIDATE:  67%|██████▋   | 213/320 [01:34<01:10,  1.52it/s]
cache-cache_patch:all-oblivious/inputs_empty_cache VALIDATE:  67%|██████▋   | 214/320 [01:34<01:12,  1.46it/s]
cache-cache_patch:all-oblivious/inputs_batch1 EXPORT:  67%|██████▋   | 214/320 [01:34<01:12,  1.46it/s]
cache-cache_patch:all-oblivious/inputs_batch1 VALIDATE:  67%|██████▋   | 214/320 [01:35<01:12,  1.46it/s]
cache-cache_patch:all-oblivious/inputs_batch1 VALIDATE:  67%|██████▋   | 215/320 [01:35<01:21,  1.29it/s]
cache-cache_patch:all-oblivious-rt/inputs EXPORT:  67%|██████▋   | 215/320 [01:35<01:21,  1.29it/s]
cache-cache_patch:all-oblivious-rt/inputs VALIDATE:  67%|██████▋   | 215/320 [01:37<01:21,  1.29it/s]
cache-cache_patch:all-oblivious-rt/inputs VALIDATE:  68%|██████▊   | 216/320 [01:37<01:57,  1.13s/it]
cache-cache_patch:all-oblivious-rt/inputs_prompt EXPORT:  68%|██████▊   | 216/320 [01:37<01:57,  1.13s/it]
cache-cache_patch:all-oblivious-rt/inputs2 EXPORT:  68%|██████▊   | 216/320 [01:37<01:57,  1.13s/it]
cache-cache_patch:all-oblivious-rt/inputs2 VALIDATE:  68%|██████▊   | 216/320 [01:38<01:57,  1.13s/it]
cache-cache_patch:all-oblivious-rt/inputs2 VALIDATE:  68%|██████▊   | 218/320 [01:38<01:25,  1.19it/s]
cache-cache_patch:all-oblivious-rt/inputs_empty_cache EXPORT:  68%|██████▊   | 218/320 [01:38<01:25,  1.19it/s]
cache-cache_patch:all-oblivious-rt/inputs_empty_cache VALIDATE:  68%|██████▊   | 218/320 [01:39<01:25,  1.19it/s]
cache-cache_patch:all-oblivious-rt/inputs_empty_cache VALIDATE:  68%|██████▊   | 219/320 [01:39<01:26,  1.17it/s]
cache-cache_patch:all-oblivious-rt/inputs_batch1 EXPORT:  68%|██████▊   | 219/320 [01:39<01:26,  1.17it/s]
cache-cache_patch:all-oblivious-rt/inputs_batch1 VALIDATE:  68%|██████▊   | 219/320 [01:40<01:26,  1.17it/s]
cache-cache_patch:all-oblivious-rt/inputs_batch1 VALIDATE:  69%|██████▉   | 220/320 [01:40<01:25,  1.17it/s]
cache-cache_patch:all-strict/inputs EXPORT:  69%|██████▉   | 220/320 [01:40<01:25,  1.17it/s]
cache-cache_patch:all-strict/inputs EXPORT:  69%|██████▉   | 221/320 [01:41<01:22,  1.19it/s]
cache-cache_patch:all-strict/inputs_prompt EXPORT:  69%|██████▉   | 221/320 [01:41<01:22,  1.19it/s]
cache-cache_patch:all-strict/inputs2 EXPORT:  69%|██████▉   | 221/320 [01:41<01:22,  1.19it/s]
cache-cache_patch:all-strict/inputs2 EXPORT:  70%|██████▉   | 223/320 [01:41<01:02,  1.55it/s]
cache-cache_patch:all-strict/inputs_empty_cache EXPORT:  70%|██████▉   | 223/320 [01:41<01:02,  1.55it/s]
cache-cache_patch:all-strict/inputs_empty_cache EXPORT:  70%|███████   | 224/320 [01:42<01:02,  1.55it/s]
cache-cache_patch:all-strict/inputs_batch1 EXPORT:  70%|███████   | 224/320 [01:42<01:02,  1.55it/s]
cache-cache_patch:all-strict/inputs_batch1 EXPORT:  70%|███████   | 225/320 [01:43<00:59,  1.59it/s]
cache-cache_patch:all-strict-rt/inputs EXPORT:  70%|███████   | 225/320 [01:43<00:59,  1.59it/s]
cache-cache_patch:all-strict-rt/inputs EXPORT:  71%|███████   | 226/320 [01:43<01:00,  1.55it/s]
cache-cache_patch:all-strict-rt/inputs_prompt EXPORT:  71%|███████   | 226/320 [01:43<01:00,  1.55it/s]
cache-cache_patch:all-strict-rt/inputs2 EXPORT:  71%|███████   | 226/320 [01:43<01:00,  1.55it/s]
cache-cache_patch:all-strict-rt/inputs2 EXPORT:  71%|███████▏  | 228/320 [01:44<00:47,  1.95it/s]
cache-cache_patch:all-strict-rt/inputs_empty_cache EXPORT:  71%|███████▏  | 228/320 [01:44<00:47,  1.95it/s]
cache-cache_patch:all-strict-rt/inputs_empty_cache EXPORT:  72%|███████▏  | 229/320 [01:45<00:49,  1.83it/s]
cache-cache_patch:all-strict-rt/inputs_batch1 EXPORT:  72%|███████▏  | 229/320 [01:45<00:49,  1.83it/s]
cache-cache_patch:all-strict-rt/inputs_batch1 EXPORT:  72%|███████▏  | 230/320 [01:45<00:50,  1.79it/s]
cache-cache_patch:all-strict-oblivious/inputs EXPORT:  72%|███████▏  | 230/320 [01:45<00:50,  1.79it/s]
cache-cache_patch:all-strict-oblivious/inputs EXPORT:  72%|███████▏  | 231/320 [01:46<00:55,  1.59it/s]
cache-cache_patch:all-strict-oblivious/inputs_prompt EXPORT:  72%|███████▏  | 231/320 [01:46<00:55,  1.59it/s]
cache-cache_patch:all-strict-oblivious/inputs2 EXPORT:  72%|███████▏  | 231/320 [01:46<00:55,  1.59it/s]
cache-cache_patch:all-strict-oblivious/inputs2 EXPORT:  73%|███████▎  | 233/320 [01:47<00:45,  1.91it/s]
cache-cache_patch:all-strict-oblivious/inputs_empty_cache EXPORT:  73%|███████▎  | 233/320 [01:47<00:45,  1.91it/s]
cache-cache_patch:all-strict-oblivious/inputs_empty_cache EXPORT:  73%|███████▎  | 234/320 [01:48<00:50,  1.69it/s]
cache-cache_patch:all-strict-oblivious/inputs_batch1 EXPORT:  73%|███████▎  | 234/320 [01:48<00:50,  1.69it/s]
cache-cache_patch:all-strict-oblivious/inputs_batch1 EXPORT:  73%|███████▎  | 235/320 [01:48<00:53,  1.59it/s]
cache-cache_patch:all-strict-oblivious-rt/inputs EXPORT:  73%|███████▎  | 235/320 [01:48<00:53,  1.59it/s]
cache-cache_patch:all-strict-oblivious-rt/inputs EXPORT:  74%|███████▍  | 236/320 [01:49<00:54,  1.53it/s]
cache-cache_patch:all-strict-oblivious-rt/inputs_prompt EXPORT:  74%|███████▍  | 236/320 [01:49<00:54,  1.53it/s]
cache-cache_patch:all-strict-oblivious-rt/inputs2 EXPORT:  74%|███████▍  | 236/320 [01:49<00:54,  1.53it/s]
cache-cache_patch:all-strict-oblivious-rt/inputs2 EXPORT:  74%|███████▍  | 238/320 [01:50<00:42,  1.94it/s]
cache-cache_patch:all-strict-oblivious-rt/inputs_empty_cache EXPORT:  74%|███████▍  | 238/320 [01:50<00:42,  1.94it/s]
cache-cache_patch:all-strict-oblivious-rt/inputs_empty_cache EXPORT:  75%|███████▍  | 239/320 [01:51<01:04,  1.26it/s]
cache-cache_patch:all-strict-oblivious-rt/inputs_batch1 EXPORT:  75%|███████▍  | 239/320 [01:51<01:04,  1.26it/s]
cache-cache_patch:all-strict-oblivious-rt/inputs_batch1 EXPORT:  75%|███████▌  | 240/320 [01:52<01:01,  1.30it/s]
cache-cache_patch:torch/inputs EXPORT:  75%|███████▌  | 240/320 [01:52<01:01,  1.30it/s]
cache-cache_patch:torch/inputs VALIDATE:  75%|███████▌  | 240/320 [01:53<01:01,  1.30it/s]
cache-cache_patch:torch/inputs VALIDATE:  75%|███████▌  | 241/320 [01:53<01:06,  1.19it/s]
cache-cache_patch:torch/inputs_prompt EXPORT:  75%|███████▌  | 241/320 [01:53<01:06,  1.19it/s]
cache-cache_patch:torch/inputs2 EXPORT:  75%|███████▌  | 241/320 [01:53<01:06,  1.19it/s]
cache-cache_patch:torch/inputs2 VALIDATE:  75%|███████▌  | 241/320 [01:54<01:06,  1.19it/s]
cache-cache_patch:torch/inputs2 VALIDATE:  76%|███████▌  | 243/320 [01:54<00:51,  1.51it/s]
cache-cache_patch:torch/inputs_empty_cache EXPORT:  76%|███████▌  | 243/320 [01:54<00:51,  1.51it/s]
cache-cache_patch:torch/inputs_empty_cache VALIDATE:  76%|███████▌  | 243/320 [01:55<00:51,  1.51it/s]
cache-cache_patch:torch/inputs_empty_cache VALIDATE:  76%|███████▋  | 244/320 [01:55<00:53,  1.43it/s]
cache-cache_patch:torch/inputs_batch1 EXPORT:  76%|███████▋  | 244/320 [01:55<00:53,  1.43it/s]
cache-cache_patch:torch/inputs_batch1 VALIDATE:  76%|███████▋  | 244/320 [01:55<00:53,  1.43it/s]
cache-cache_patch:torch/inputs_batch1 VALIDATE:  77%|███████▋  | 245/320 [01:55<00:52,  1.44it/s]
cache-cache_patch:torch-rt/inputs EXPORT:  77%|███████▋  | 245/320 [01:55<00:52,  1.44it/s]
cache-cache_patch:torch-rt/inputs VALIDATE:  77%|███████▋  | 245/320 [01:56<00:52,  1.44it/s]
cache-cache_patch:torch-rt/inputs VALIDATE:  77%|███████▋  | 246/320 [01:56<00:54,  1.35it/s]
cache-cache_patch:torch-rt/inputs_prompt EXPORT:  77%|███████▋  | 246/320 [01:56<00:54,  1.35it/s]
cache-cache_patch:torch-rt/inputs2 EXPORT:  77%|███████▋  | 246/320 [01:56<00:54,  1.35it/s]
cache-cache_patch:torch-rt/inputs2 VALIDATE:  77%|███████▋  | 246/320 [01:57<00:54,  1.35it/s]
cache-cache_patch:torch-rt/inputs2 VALIDATE:  78%|███████▊  | 248/320 [01:57<00:44,  1.63it/s]
cache-cache_patch:torch-rt/inputs_empty_cache EXPORT:  78%|███████▊  | 248/320 [01:57<00:44,  1.63it/s]
cache-cache_patch:torch-rt/inputs_empty_cache VALIDATE:  78%|███████▊  | 248/320 [01:58<00:44,  1.63it/s]
cache-cache_patch:torch-rt/inputs_empty_cache VALIDATE:  78%|███████▊  | 249/320 [01:58<00:46,  1.52it/s]
cache-cache_patch:torch-rt/inputs_batch1 EXPORT:  78%|███████▊  | 249/320 [01:58<00:46,  1.52it/s]
cache-cache_patch:torch-rt/inputs_batch1 VALIDATE:  78%|███████▊  | 249/320 [01:59<00:46,  1.52it/s]
cache-cache_patch:torch-rt/inputs_batch1 VALIDATE:  78%|███████▊  | 250/320 [01:59<00:45,  1.52it/s]
cache-cache_patch:torch-oblivious/inputs EXPORT:  78%|███████▊  | 250/320 [01:59<00:45,  1.52it/s]
cache-cache_patch:torch-oblivious/inputs VALIDATE:  78%|███████▊  | 250/320 [02:00<00:45,  1.52it/s]
cache-cache_patch:torch-oblivious/inputs VALIDATE:  78%|███████▊  | 251/320 [02:00<00:50,  1.36it/s]
cache-cache_patch:torch-oblivious/inputs_prompt EXPORT:  78%|███████▊  | 251/320 [02:00<00:50,  1.36it/s]
cache-cache_patch:torch-oblivious/inputs2 EXPORT:  78%|███████▊  | 251/320 [02:00<00:50,  1.36it/s]
cache-cache_patch:torch-oblivious/inputs2 VALIDATE:  78%|███████▊  | 251/320 [02:00<00:50,  1.36it/s]
cache-cache_patch:torch-oblivious/inputs2 VALIDATE:  79%|███████▉  | 253/320 [02:01<00:41,  1.62it/s]
cache-cache_patch:torch-oblivious/inputs_empty_cache EXPORT:  79%|███████▉  | 253/320 [02:01<00:41,  1.62it/s]
cache-cache_patch:torch-oblivious/inputs_empty_cache VALIDATE:  79%|███████▉  | 253/320 [02:01<00:41,  1.62it/s]
cache-cache_patch:torch-oblivious/inputs_empty_cache VALIDATE:  79%|███████▉  | 254/320 [02:02<00:46,  1.42it/s]
cache-cache_patch:torch-oblivious/inputs_batch1 EXPORT:  79%|███████▉  | 254/320 [02:02<00:46,  1.42it/s]
cache-cache_patch:torch-oblivious/inputs_batch1 VALIDATE:  79%|███████▉  | 254/320 [02:02<00:46,  1.42it/s]
cache-cache_patch:torch-oblivious/inputs_batch1 VALIDATE:  80%|███████▉  | 255/320 [02:02<00:48,  1.33it/s]
cache-cache_patch:torch-oblivious-rt/inputs EXPORT:  80%|███████▉  | 255/320 [02:02<00:48,  1.33it/s]
cache-cache_patch:torch-oblivious-rt/inputs VALIDATE:  80%|███████▉  | 255/320 [02:03<00:48,  1.33it/s]
cache-cache_patch:torch-oblivious-rt/inputs VALIDATE:  80%|████████  | 256/320 [02:03<00:50,  1.26it/s]
cache-cache_patch:torch-oblivious-rt/inputs_prompt EXPORT:  80%|████████  | 256/320 [02:03<00:50,  1.26it/s]
cache-cache_patch:torch-oblivious-rt/inputs2 EXPORT:  80%|████████  | 256/320 [02:03<00:50,  1.26it/s]
cache-cache_patch:torch-oblivious-rt/inputs2 VALIDATE:  80%|████████  | 256/320 [02:04<00:50,  1.26it/s]
cache-cache_patch:torch-oblivious-rt/inputs2 VALIDATE:  81%|████████  | 258/320 [02:04<00:40,  1.55it/s]
cache-cache_patch:torch-oblivious-rt/inputs_empty_cache EXPORT:  81%|████████  | 258/320 [02:04<00:40,  1.55it/s]
cache-cache_patch:torch-oblivious-rt/inputs_empty_cache VALIDATE:  81%|████████  | 258/320 [02:05<00:40,  1.55it/s]
cache-cache_patch:torch-oblivious-rt/inputs_empty_cache VALIDATE:  81%|████████  | 259/320 [02:05<00:42,  1.43it/s]
cache-cache_patch:torch-oblivious-rt/inputs_batch1 EXPORT:  81%|████████  | 259/320 [02:05<00:42,  1.43it/s]
cache-cache_patch:torch-oblivious-rt/inputs_batch1 VALIDATE:  81%|████████  | 259/320 [02:06<00:42,  1.43it/s]
cache-cache_patch:torch-oblivious-rt/inputs_batch1 VALIDATE:  81%|████████▏ | 260/320 [02:06<00:44,  1.34it/s]
cache-cache_patch:torch-strict/inputs EXPORT:  81%|████████▏ | 260/320 [02:06<00:44,  1.34it/s]
cache-cache_patch:torch-strict/inputs EXPORT:  82%|████████▏ | 261/320 [02:07<00:50,  1.17it/s]
cache-cache_patch:torch-strict/inputs_prompt EXPORT:  82%|████████▏ | 261/320 [02:07<00:50,  1.17it/s]
cache-cache_patch:torch-strict/inputs2 EXPORT:  82%|████████▏ | 261/320 [02:07<00:50,  1.17it/s]
cache-cache_patch:torch-strict/inputs2 EXPORT:  82%|████████▏ | 263/320 [02:08<00:42,  1.36it/s]
cache-cache_patch:torch-strict/inputs_empty_cache EXPORT:  82%|████████▏ | 263/320 [02:08<00:42,  1.36it/s]
cache-cache_patch:torch-strict/inputs_empty_cache EXPORT:  82%|████████▎ | 264/320 [02:11<01:02,  1.12s/it]
cache-cache_patch:torch-strict/inputs_batch1 EXPORT:  82%|████████▎ | 264/320 [02:11<01:02,  1.12s/it]
cache-cache_patch:torch-strict/inputs_batch1 EXPORT:  83%|████████▎ | 265/320 [02:12<01:10,  1.29s/it]
cache-cache_patch:torch-strict-rt/inputs EXPORT:  83%|████████▎ | 265/320 [02:12<01:10,  1.29s/it]
cache-cache_patch:torch-strict-rt/inputs EXPORT:  83%|████████▎ | 266/320 [02:15<01:20,  1.48s/it]
cache-cache_patch:torch-strict-rt/inputs_prompt EXPORT:  83%|████████▎ | 266/320 [02:15<01:20,  1.48s/it]
cache-cache_patch:torch-strict-rt/inputs2 EXPORT:  83%|████████▎ | 266/320 [02:15<01:20,  1.48s/it]
cache-cache_patch:torch-strict-rt/inputs2 EXPORT:  84%|████████▍ | 268/320 [02:16<01:06,  1.27s/it]
cache-cache_patch:torch-strict-rt/inputs_empty_cache EXPORT:  84%|████████▍ | 268/320 [02:16<01:06,  1.27s/it]
cache-cache_patch:torch-strict-rt/inputs_empty_cache EXPORT:  84%|████████▍ | 269/320 [02:19<01:15,  1.47s/it]
cache-cache_patch:torch-strict-rt/inputs_batch1 EXPORT:  84%|████████▍ | 269/320 [02:19<01:15,  1.47s/it]
cache-cache_patch:torch-strict-rt/inputs_batch1 EXPORT:  84%|████████▍ | 270/320 [02:20<01:17,  1.54s/it]
cache-cache_patch:torch-strict-oblivious/inputs EXPORT:  84%|████████▍ | 270/320 [02:20<01:17,  1.54s/it]
cache-cache_patch:torch-strict-oblivious/inputs EXPORT:  85%|████████▍ | 271/320 [02:22<01:13,  1.50s/it]
cache-cache_patch:torch-strict-oblivious/inputs_prompt EXPORT:  85%|████████▍ | 271/320 [02:22<01:13,  1.50s/it]
cache-cache_patch:torch-strict-oblivious/inputs2 EXPORT:  85%|████████▍ | 271/320 [02:22<01:13,  1.50s/it]
cache-cache_patch:torch-strict-oblivious/inputs2 EXPORT:  85%|████████▌ | 273/320 [02:23<00:53,  1.15s/it]
cache-cache_patch:torch-strict-oblivious/inputs_empty_cache EXPORT:  85%|████████▌ | 273/320 [02:23<00:53,  1.15s/it]
cache-cache_patch:torch-strict-oblivious/inputs_empty_cache EXPORT:  86%|████████▌ | 274/320 [02:24<00:53,  1.17s/it]
cache-cache_patch:torch-strict-oblivious/inputs_batch1 EXPORT:  86%|████████▌ | 274/320 [02:24<00:53,  1.17s/it]
cache-cache_patch:torch-strict-oblivious/inputs_batch1 EXPORT:  86%|████████▌ | 275/320 [02:26<01:02,  1.40s/it]
cache-cache_patch:torch-strict-oblivious-rt/inputs EXPORT:  86%|████████▌ | 275/320 [02:26<01:02,  1.40s/it]
cache-cache_patch:torch-strict-oblivious-rt/inputs EXPORT:  86%|████████▋ | 276/320 [02:29<01:10,  1.60s/it]
cache-cache_patch:torch-strict-oblivious-rt/inputs_prompt EXPORT:  86%|████████▋ | 276/320 [02:29<01:10,  1.60s/it]
cache-cache_patch:torch-strict-oblivious-rt/inputs2 EXPORT:  86%|████████▋ | 276/320 [02:29<01:10,  1.60s/it]
cache-cache_patch:torch-strict-oblivious-rt/inputs2 EXPORT:  87%|████████▋ | 278/320 [02:31<00:58,  1.39s/it]
cache-cache_patch:torch-strict-oblivious-rt/inputs_empty_cache EXPORT:  87%|████████▋ | 278/320 [02:31<00:58,  1.39s/it]
cache-cache_patch:torch-strict-oblivious-rt/inputs_empty_cache EXPORT:  87%|████████▋ | 279/320 [02:35<01:19,  1.93s/it]
cache-cache_patch:torch-strict-oblivious-rt/inputs_batch1 EXPORT:  87%|████████▋ | 279/320 [02:35<01:19,  1.93s/it]
cache-cache_patch:torch-strict-oblivious-rt/inputs_batch1 EXPORT:  88%|████████▊ | 280/320 [02:37<01:18,  1.95s/it]
cache-cache_patch:transformers/inputs EXPORT:  88%|████████▊ | 280/320 [02:37<01:18,  1.95s/it]
cache-cache_patch:transformers/inputs VALIDATE:  88%|████████▊ | 280/320 [02:37<01:18,  1.95s/it]
cache-cache_patch:transformers/inputs VALIDATE:  88%|████████▊ | 281/320 [02:38<01:07,  1.72s/it]
cache-cache_patch:transformers/inputs_prompt EXPORT:  88%|████████▊ | 281/320 [02:38<01:07,  1.72s/it]
cache-cache_patch:transformers/inputs2 EXPORT:  88%|████████▊ | 281/320 [02:38<01:07,  1.72s/it]
cache-cache_patch:transformers/inputs2 VALIDATE:  88%|████████▊ | 281/320 [02:38<01:07,  1.72s/it]
cache-cache_patch:transformers/inputs2 VALIDATE:  88%|████████▊ | 283/320 [02:39<00:44,  1.19s/it]
cache-cache_patch:transformers/inputs_empty_cache EXPORT:  88%|████████▊ | 283/320 [02:39<00:44,  1.19s/it]
cache-cache_patch:transformers/inputs_empty_cache EXPORT:  89%|████████▉ | 284/320 [02:39<00:39,  1.09s/it]
cache-cache_patch:transformers/inputs_batch1 EXPORT:  89%|████████▉ | 284/320 [02:39<00:39,  1.09s/it]
cache-cache_patch:transformers/inputs_batch1 EXPORT:  89%|████████▉ | 285/320 [02:40<00:34,  1.02it/s]
cache-cache_patch:transformers-rt/inputs EXPORT:  89%|████████▉ | 285/320 [02:40<00:34,  1.02it/s]
cache-cache_patch:transformers-rt/inputs VALIDATE:  89%|████████▉ | 285/320 [02:41<00:34,  1.02it/s]
cache-cache_patch:transformers-rt/inputs VALIDATE:  89%|████████▉ | 286/320 [02:41<00:33,  1.03it/s]
cache-cache_patch:transformers-rt/inputs_prompt EXPORT:  89%|████████▉ | 286/320 [02:41<00:33,  1.03it/s]
cache-cache_patch:transformers-rt/inputs2 EXPORT:  89%|████████▉ | 286/320 [02:41<00:33,  1.03it/s]
cache-cache_patch:transformers-rt/inputs2 VALIDATE:  89%|████████▉ | 286/320 [02:42<00:33,  1.03it/s]
cache-cache_patch:transformers-rt/inputs2 VALIDATE:  90%|█████████ | 288/320 [02:42<00:23,  1.35it/s]
cache-cache_patch:transformers-rt/inputs_empty_cache EXPORT:  90%|█████████ | 288/320 [02:42<00:23,  1.35it/s]
cache-cache_patch:transformers-rt/inputs_empty_cache EXPORT:  90%|█████████ | 289/320 [02:43<00:23,  1.31it/s]
cache-cache_patch:transformers-rt/inputs_batch1 EXPORT:  90%|█████████ | 289/320 [02:43<00:23,  1.31it/s]
cache-cache_patch:transformers-rt/inputs_batch1 EXPORT:  91%|█████████ | 290/320 [02:43<00:22,  1.36it/s]
cache-cache_patch:transformers-oblivious/inputs EXPORT:  91%|█████████ | 290/320 [02:43<00:22,  1.36it/s]
cache-cache_patch:transformers-oblivious/inputs VALIDATE:  91%|█████████ | 290/320 [02:44<00:22,  1.36it/s]
cache-cache_patch:transformers-oblivious/inputs VALIDATE:  91%|█████████ | 291/320 [02:44<00:22,  1.31it/s]
cache-cache_patch:transformers-oblivious/inputs_prompt EXPORT:  91%|█████████ | 291/320 [02:44<00:22,  1.31it/s]
cache-cache_patch:transformers-oblivious/inputs2 EXPORT:  91%|█████████ | 291/320 [02:44<00:22,  1.31it/s]
cache-cache_patch:transformers-oblivious/inputs2 VALIDATE:  91%|█████████ | 291/320 [02:45<00:22,  1.31it/s]
cache-cache_patch:transformers-oblivious/inputs2 VALIDATE:  92%|█████████▏| 293/320 [02:45<00:16,  1.63it/s]
cache-cache_patch:transformers-oblivious/inputs_empty_cache EXPORT:  92%|█████████▏| 293/320 [02:45<00:16,  1.63it/s]
cache-cache_patch:transformers-oblivious/inputs_empty_cache VALIDATE:  92%|█████████▏| 293/320 [02:46<00:16,  1.63it/s]
cache-cache_patch:transformers-oblivious/inputs_empty_cache VALIDATE:  92%|█████████▏| 294/320 [02:46<00:17,  1.50it/s]
cache-cache_patch:transformers-oblivious/inputs_batch1 EXPORT:  92%|█████████▏| 294/320 [02:46<00:17,  1.50it/s]
cache-cache_patch:transformers-oblivious/inputs_batch1 VALIDATE:  92%|█████████▏| 294/320 [02:47<00:17,  1.50it/s]
cache-cache_patch:transformers-oblivious/inputs_batch1 VALIDATE:  92%|█████████▏| 295/320 [02:47<00:17,  1.40it/s]
cache-cache_patch:transformers-oblivious-rt/inputs EXPORT:  92%|█████████▏| 295/320 [02:47<00:17,  1.40it/s]
cache-cache_patch:transformers-oblivious-rt/inputs VALIDATE:  92%|█████████▏| 295/320 [02:47<00:17,  1.40it/s]
cache-cache_patch:transformers-oblivious-rt/inputs VALIDATE:  92%|█████████▎| 296/320 [02:47<00:17,  1.34it/s]
cache-cache_patch:transformers-oblivious-rt/inputs_prompt EXPORT:  92%|█████████▎| 296/320 [02:47<00:17,  1.34it/s]
cache-cache_patch:transformers-oblivious-rt/inputs2 EXPORT:  92%|█████████▎| 296/320 [02:47<00:17,  1.34it/s]
cache-cache_patch:transformers-oblivious-rt/inputs2 VALIDATE:  92%|█████████▎| 296/320 [02:48<00:17,  1.34it/s]
cache-cache_patch:transformers-oblivious-rt/inputs2 VALIDATE:  93%|█████████▎| 298/320 [02:48<00:13,  1.61it/s]
cache-cache_patch:transformers-oblivious-rt/inputs_empty_cache EXPORT:  93%|█████████▎| 298/320 [02:48<00:13,  1.61it/s]
cache-cache_patch:transformers-oblivious-rt/inputs_empty_cache VALIDATE:  93%|█████████▎| 298/320 [02:49<00:13,  1.61it/s]
cache-cache_patch:transformers-oblivious-rt/inputs_empty_cache VALIDATE:  93%|█████████▎| 299/320 [02:49<00:14,  1.46it/s]
cache-cache_patch:transformers-oblivious-rt/inputs_batch1 EXPORT:  93%|█████████▎| 299/320 [02:49<00:14,  1.46it/s]
cache-cache_patch:transformers-oblivious-rt/inputs_batch1 VALIDATE:  93%|█████████▎| 299/320 [02:50<00:14,  1.46it/s]
cache-cache_patch:transformers-oblivious-rt/inputs_batch1 VALIDATE:  94%|█████████▍| 300/320 [02:50<00:14,  1.37it/s]
cache-cache_patch:transformers-strict/inputs EXPORT:  94%|█████████▍| 300/320 [02:50<00:14,  1.37it/s]
cache-cache_patch:transformers-strict/inputs EXPORT:  94%|█████████▍| 301/320 [02:51<00:14,  1.31it/s]
cache-cache_patch:transformers-strict/inputs_prompt EXPORT:  94%|█████████▍| 301/320 [02:51<00:14,  1.31it/s]
cache-cache_patch:transformers-strict/inputs2 EXPORT:  94%|█████████▍| 301/320 [02:51<00:14,  1.31it/s]
cache-cache_patch:transformers-strict/inputs2 EXPORT:  95%|█████████▍| 303/320 [02:52<00:10,  1.67it/s]
cache-cache_patch:transformers-strict/inputs_empty_cache EXPORT:  95%|█████████▍| 303/320 [02:52<00:10,  1.67it/s]
cache-cache_patch:transformers-strict/inputs_empty_cache EXPORT:  95%|█████████▌| 304/320 [02:52<00:09,  1.62it/s]
cache-cache_patch:transformers-strict/inputs_batch1 EXPORT:  95%|█████████▌| 304/320 [02:52<00:09,  1.62it/s]
cache-cache_patch:transformers-strict/inputs_batch1 EXPORT:  95%|█████████▌| 305/320 [02:53<00:09,  1.62it/s]
cache-cache_patch:transformers-strict-rt/inputs EXPORT:  95%|█████████▌| 305/320 [02:53<00:09,  1.62it/s]
cache-cache_patch:transformers-strict-rt/inputs EXPORT:  96%|█████████▌| 306/320 [02:54<00:08,  1.56it/s]
cache-cache_patch:transformers-strict-rt/inputs_prompt EXPORT:  96%|█████████▌| 306/320 [02:54<00:08,  1.56it/s]
cache-cache_patch:transformers-strict-rt/inputs2 EXPORT:  96%|█████████▌| 306/320 [02:54<00:08,  1.56it/s]
cache-cache_patch:transformers-strict-rt/inputs2 EXPORT:  96%|█████████▋| 308/320 [02:55<00:06,  1.88it/s]
cache-cache_patch:transformers-strict-rt/inputs_empty_cache EXPORT:  96%|█████████▋| 308/320 [02:55<00:06,  1.88it/s]
cache-cache_patch:transformers-strict-rt/inputs_empty_cache EXPORT:  97%|█████████▋| 309/320 [02:57<00:10,  1.02it/s]
cache-cache_patch:transformers-strict-rt/inputs_batch1 EXPORT:  97%|█████████▋| 309/320 [02:57<00:10,  1.02it/s]
cache-cache_patch:transformers-strict-rt/inputs_batch1 EXPORT:  97%|█████████▋| 310/320 [02:58<00:08,  1.14it/s]
cache-cache_patch:transformers-strict-oblivious/inputs EXPORT:  97%|█████████▋| 310/320 [02:58<00:08,  1.14it/s]
cache-cache_patch:transformers-strict-oblivious/inputs EXPORT:  97%|█████████▋| 311/320 [02:58<00:07,  1.19it/s]
cache-cache_patch:transformers-strict-oblivious/inputs_prompt EXPORT:  97%|█████████▋| 311/320 [02:58<00:07,  1.19it/s]
cache-cache_patch:transformers-strict-oblivious/inputs2 EXPORT:  97%|█████████▋| 311/320 [02:58<00:07,  1.19it/s]
cache-cache_patch:transformers-strict-oblivious/inputs2 EXPORT:  98%|█████████▊| 313/320 [02:59<00:04,  1.54it/s]
cache-cache_patch:transformers-strict-oblivious/inputs_empty_cache EXPORT:  98%|█████████▊| 313/320 [02:59<00:04,  1.54it/s]
cache-cache_patch:transformers-strict-oblivious/inputs_empty_cache EXPORT:  98%|█████████▊| 314/320 [03:00<00:03,  1.53it/s]
cache-cache_patch:transformers-strict-oblivious/inputs_batch1 EXPORT:  98%|█████████▊| 314/320 [03:00<00:03,  1.53it/s]
cache-cache_patch:transformers-strict-oblivious/inputs_batch1 EXPORT:  98%|█████████▊| 315/320 [03:00<00:03,  1.65it/s]
cache-cache_patch:transformers-strict-oblivious-rt/inputs EXPORT:  98%|█████████▊| 315/320 [03:00<00:03,  1.65it/s]
cache-cache_patch:transformers-strict-oblivious-rt/inputs EXPORT:  99%|█████████▉| 316/320 [03:01<00:02,  1.56it/s]
cache-cache_patch:transformers-strict-oblivious-rt/inputs_prompt EXPORT:  99%|█████████▉| 316/320 [03:01<00:02,  1.56it/s]
cache-cache_patch:transformers-strict-oblivious-rt/inputs2 EXPORT:  99%|█████████▉| 316/320 [03:01<00:02,  1.56it/s]
cache-cache_patch:transformers-strict-oblivious-rt/inputs2 EXPORT:  99%|█████████▉| 318/320 [03:02<00:01,  1.91it/s]
cache-cache_patch:transformers-strict-oblivious-rt/inputs_empty_cache EXPORT:  99%|█████████▉| 318/320 [03:02<00:01,  1.91it/s]
cache-cache_patch:transformers-strict-oblivious-rt/inputs_empty_cache EXPORT: 100%|█████████▉| 319/320 [03:03<00:00,  1.68it/s]
cache-cache_patch:transformers-strict-oblivious-rt/inputs_batch1 EXPORT: 100%|█████████▉| 319/320 [03:03<00:00,  1.68it/s]
cache-cache_patch:transformers-strict-oblivious-rt/inputs_batch1 EXPORT: 100%|██████████| 320/320 [03:03<00:00,  1.59it/s]
cache-cache_patch:transformers-strict-oblivious-rt/inputs_batch1 EXPORT: 100%|██████████| 320/320 [03:03<00:00,  1.74it/s]

Let’s save the results.

df = pandas.DataFrame(results)
df.to_excel("plot_export_tiny_llm_dim01.xlsx")
df

	cache	cache_patch	strict	oblivious	rt	export_with	EXPORT	run_with	WORKS	ERR-RUN	ERR-EXPORT
0	0	0	0	0	0	inputs	1	inputs	1.0	NaN	NaN
1	0	0	0	0	0	inputs	1	inputs_prompt	0.0	Ran into a kwarg keyword mismatch: Got the fol...	NaN
2	0	0	0	0	0	inputs	1	inputs2	1.0	NaN	NaN
3	0	0	0	0	0	inputs	1	inputs_empty_cache	0.0	Guard failed: attention_mask.size()[1] >= 4	NaN
4	0	0	0	0	0	inputs	1	inputs_batch1	1.0	NaN	NaN
...	...	...	...	...	...	...	...	...	...	...	...
467	1	transformers	1	1	1	inputs	0	NaN	NaN	NaN	Failed to convert args/kwargs to proxy
468	1	transformers	1	1	1	inputs_prompt	0	NaN	NaN	NaN	When `dynamic_shapes` is specified as a dict, ...
469	1	transformers	1	1	1	inputs2	0	NaN	NaN	NaN	Failed to convert args/kwargs to proxy
470	1	transformers	1	1	1	inputs_empty_cache	0	NaN	NaN	NaN	Failed to convert args/kwargs to proxy
471	1	transformers	1	1	1	inputs_batch1	0	NaN	NaN	NaN	Failed to convert args/kwargs to proxy

472 rows × 11 columns

no_export = df[df.EXPORT == 0]
no_export.to_excel("plot_export_tiny_llm_dim01.no_export.xlsx")
no_export

	cache	cache_patch	strict	oblivious	rt	export_with	EXPORT	run_with	WORKS	ERR-RUN	ERR-EXPORT
5	0	0	0	0	0	inputs_prompt	0	NaN	NaN	NaN	When `dynamic_shapes` is specified as a dict, ...
11	0	0	0	0	0	inputs_empty_cache	0	NaN	NaN	NaN	Found the following conflicts between user-spe...
12	0	0	0	0	0	inputs_batch1	0	NaN	NaN	NaN	Found the following conflicts between user-spe...
18	0	0	0	0	1	inputs_prompt	0	NaN	NaN	NaN	When `dynamic_shapes` is specified as a dict, ...
24	0	0	0	0	1	inputs_empty_cache	0	NaN	NaN	NaN	Found the following conflicts between user-spe...
...	...	...	...	...	...	...	...	...	...	...	...
467	1	transformers	1	1	1	inputs	0	NaN	NaN	NaN	Failed to convert args/kwargs to proxy
468	1	transformers	1	1	1	inputs_prompt	0	NaN	NaN	NaN	When `dynamic_shapes` is specified as a dict, ...
469	1	transformers	1	1	1	inputs2	0	NaN	NaN	NaN	Failed to convert args/kwargs to proxy
470	1	transformers	1	1	1	inputs_empty_cache	0	NaN	NaN	NaN	Failed to convert args/kwargs to proxy
471	1	transformers	1	1	1	inputs_batch1	0	NaN	NaN	NaN	Failed to convert args/kwargs to proxy

132 rows × 11 columns

The validation failures.

invalid = df[(df.EXPORT == 1) & (df.WORKS == 0)].pivot(
    index=["cache", "cache_patch", "strict", "oblivious", "rt", "export_with"],
    columns=["run_with"],
    values=["WORKS", "ERR-RUN"],
)
invalid.to_excel("plot_export_tiny_llm_dim01.invalid.xlsx")
invalid

						WORKS					ERR-RUN
					run_with	inputs	inputs2	inputs_batch1	inputs_empty_cache	inputs_prompt	inputs	inputs2	inputs_batch1	inputs_empty_cache	inputs_prompt
cache	cache_patch	strict	oblivious	rt	export_with
0	0	0	0	0	inputs	NaN	NaN	NaN	0.0	0.0	NaN	NaN	NaN	Guard failed: attention_mask.size()[1] >= 4	Ran into a kwarg keyword mismatch: Got the fol...
				0	inputs2	NaN	NaN	NaN	0.0	0.0	NaN	NaN	NaN	Guard failed: attention_mask.size()[1] >= 4	Ran into a kwarg keyword mismatch: Got the fol...
				1	inputs	NaN	NaN	NaN	0.0	0.0	NaN	NaN	NaN	Guard failed: attention_mask.size()[1] >= 4	Ran into a kwarg keyword mismatch: Got the fol...
				1	inputs2	NaN	NaN	NaN	0.0	0.0	NaN	NaN	NaN	Guard failed: attention_mask.size()[1] >= 4	Ran into a kwarg keyword mismatch: Got the fol...
			1	0	inputs	NaN	NaN	0.0	NaN	0.0	NaN	NaN	Guard failed: position_ids.size()[0] != 1	NaN	Ran into a kwarg keyword mismatch: Got the fol...
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
1	transformers	0	1	0	inputs_empty_cache	NaN	NaN	0.0	NaN	0.0	NaN	NaN	Guard failed: position_ids.size()[0] != 1	NaN	Ran into a kwarg keyword mismatch: Got the fol...
				1	inputs	NaN	NaN	0.0	NaN	0.0	NaN	NaN	Runtime assertion failed for expression Ne(s44...	NaN	Ran into a kwarg keyword mismatch: Got the fol...
					inputs2	NaN	NaN	0.0	NaN	0.0	NaN	NaN	Runtime assertion failed for expression Ne(s44...	NaN	Ran into a kwarg keyword mismatch: Got the fol...
					inputs_batch1	0.0	0.0	NaN	0.0	0.0	Runtime assertion failed for expression Eq(s44...	Guard failed: input_ids.size()[0] <= 2	NaN	Runtime assertion failed for expression Eq(s44...	Ran into a kwarg keyword mismatch: Got the fol...
					inputs_empty_cache	NaN	NaN	0.0	NaN	0.0	NaN	NaN	Runtime assertion failed for expression Ne(s44...	NaN	Ran into a kwarg keyword mismatch: Got the fol...

68 rows × 10 columns

success = df[(df.EXPORT == 1) & (df.WORKS == 1)].pivot(
    index=["cache", "cache_patch", "strict", "oblivious", "rt", "export_with"],
    columns=["run_with"],
    values=["WORKS"],
)
success.to_excel("plot_export_tiny_llm_dim01.success.xlsx")
success

						WORKS
					run_with	inputs	inputs2	inputs_batch1	inputs_empty_cache
cache	cache_patch	strict	oblivious	rt	export_with
0	0	0	0	0	inputs	1.0	1.0	1.0	NaN
				0	inputs2	1.0	1.0	1.0	NaN
				1	inputs	1.0	1.0	1.0	NaN
				1	inputs2	1.0	1.0	1.0	NaN
			1	0	inputs	1.0	1.0	NaN	1.0
...	...	...	...	...	...	...	...	...	...
1	transformers	0	1	0	inputs_empty_cache	1.0	1.0	NaN	1.0
				1	inputs	1.0	1.0	NaN	1.0
					inputs2	1.0	1.0	NaN	1.0
					inputs_batch1	NaN	NaN	1.0	NaN
					inputs_empty_cache	1.0	1.0	NaN	1.0

68 rows × 4 columns

If you have any error, then look at example Export Tiny-LLM with patches.

doc.plot_legend("Tiny-LLM\nexport with\ndimension in {0,1}", "torch.export.export", "tomato")

Total running time of the script: (3 minutes 4.058 seconds)

Related examples

Export with dynamic dimensions in {0,1} into ONNX

Export with dynamic dimensions in {0,1} into ONNX (custom)

Export microsoft/phi-2

Gallery generated by Sphinx-Gallery

Export with dynamic dimensions in {0,1}¶

Available input sets¶

Export with options¶

The main loop¶

Export with dynamic dimensions in `{0,1}`¶