Measuring onnxruntime performance against a cython binding

The following code measures the performance of the python bindings against a cython binding. The time spent in it is not significant when the computation is huge but it may be for small matrices.

import numpy
from pandas import DataFrame
import matplotlib.pyplot as plt
from tqdm import tqdm
from onnx import numpy_helper, TensorProto
from onnx.helper import (
    make_model,
    make_node,
    make_graph,
    make_tensor_value_info,
    make_opsetid,
)
from onnx.checker import check_model
from onnxruntime import InferenceSession
from onnx_extended.ortcy.wrap.ortinf import OrtSession
from onnx_extended.args import get_parsed_args
from onnx_extended.ext_test_case import measure_time, unit_test_going


script_args = get_parsed_args(
    "plot_bench_cypy_ort",
    description=__doc__,
    dims=(
        "1,10" if unit_test_going() else "1,10,100,1000",
        "square matrix dimensions to try, comma separated values",
    ),
    expose="repeat,number",
)

A simple onnx model

A = numpy_helper.from_array(numpy.array([1], dtype=numpy.float32), name="A")
X = make_tensor_value_info("X", TensorProto.FLOAT, [None, None])
Y = make_tensor_value_info("Y", TensorProto.FLOAT, [None, None])
node1 = make_node("Add", ["X", "A"], ["Y"])
graph = make_graph([node1], "+1", [X], [Y], [A])
onnx_model = make_model(graph, opset_imports=[make_opsetid("", 18)], ir_version=8)
check_model(onnx_model)

Two python bindings on CPU

sess_ort = InferenceSession(
    onnx_model.SerializeToString(), providers=["CPUExecutionProvider"]
)
sess_ext = OrtSession(onnx_model.SerializeToString())

x = numpy.random.randn(10, 10).astype(numpy.float32)
y = x + 1

y_ort = sess_ort.run(None, {"X": x})[0]
y_ext = sess_ext.run([x])[0]

d_ort = numpy.abs(y_ort - y).sum()
d_ext = numpy.abs(y_ext - y).sum()
print(f"Discrepancies: d_ort={d_ort}, d_ext={d_ext}")

Discrepancies: d_ort=0.0, d_ext=0.0

Time measurement

run_1_1 is a specific implementation when there is only 1 input and output.

t_ort = measure_time(lambda: sess_ort.run(None, {"X": x})[0], number=200, repeat=100)
print(f"t_ort={t_ort}")

t_ext = measure_time(lambda: sess_ext.run([x])[0], number=200, repeat=100)
print(f"t_ext={t_ext}")

t_ext2 = measure_time(lambda: sess_ext.run_1_1(x), number=200, repeat=100)
print(f"t_ext2={t_ext2}")

t_ort={'average': np.float64(6.816206299890837e-06), 'deviation': np.float64(4.8577671754785265e-06), 'min_exec': np.float64(4.879819998677704e-06), 'max_exec': np.float64(3.079738499764062e-05), 'repeat': 100, 'number': 200, 'ttime': np.float64(0.0006816206299890838), 'context_size': 64, 'warmup_time': 9.300300007453188e-05}
t_ext={'average': np.float64(9.220920799907617e-06), 'deviation': np.float64(5.9112694677095366e-06), 'min_exec': np.float64(4.495544999372214e-06), 'max_exec': np.float64(4.499776499869768e-05), 'repeat': 100, 'number': 200, 'ttime': np.float64(0.0009220920799907616), 'context_size': 64, 'warmup_time': 0.0005774110004495014}
t_ext2={'average': np.float64(5.045960999950692e-06), 'deviation': np.float64(1.491688519781219e-06), 'min_exec': np.float64(3.978839999945194e-06), 'max_exec': np.float64(1.0508724999453989e-05), 'repeat': 100, 'number': 200, 'ttime': np.float64(0.0005045960999950693), 'context_size': 64, 'warmup_time': 0.00011649099997157464}

Benchmark

dims = [int(i) for i in script_args.dims.split(",")]

data = []
for dim in tqdm(dims):
    if dim < 1000:
        number, repeat = script_args.number, script_args.repeat
    else:
        number, repeat = script_args.number * 5, script_args.repeat * 5
    x = numpy.random.randn(dim, dim).astype(numpy.float32)
    t_ort = measure_time(
        lambda x=x: sess_ort.run(None, {"X": x})[0], number=number, repeat=50
    )
    t_ort["name"] = "ort"
    t_ort["dim"] = dim
    data.append(t_ort)

    t_ext = measure_time(lambda x=x: sess_ext.run([x])[0], number=number, repeat=repeat)
    t_ext["name"] = "ext"
    t_ext["dim"] = dim
    data.append(t_ext)

    t_ext2 = measure_time(lambda x=x: sess_ext.run_1_1(x), number=number, repeat=repeat)
    t_ext2["name"] = "ext_1_1"
    t_ext2["dim"] = dim
    data.append(t_ext2)

    if unit_test_going() and dim >= 10:
        break


df = DataFrame(data)
df

  0%|          | 0/4 [00:00<?, ?it/s]
100%|██████████| 4/4 [00:00<00:00,  4.75it/s]
100%|██████████| 4/4 [00:00<00:00,  4.75it/s]

	average	deviation	min_exec	max_exec	repeat	number	ttime	context_size	warmup_time	name	dim
0	0.000005	1.440359e-06	0.000005	0.000014	50	10	0.000265	64	0.000181	ort	1
1	0.000006	1.915202e-06	0.000005	0.000011	10	10	0.000056	64	0.001632	ext	1
2	0.000004	1.165294e-07	0.000004	0.000005	10	10	0.000043	64	0.000023	ext_1_1	1
3	0.000006	1.054422e-06	0.000005	0.000011	50	10	0.000296	64	0.000075	ort	10
4	0.000007	2.343609e-07	0.000007	0.000008	10	10	0.000072	64	0.000104	ext	10
5	0.000005	8.849431e-07	0.000004	0.000007	10	10	0.000045	64	0.000022	ext_1_1	10
6	0.000007	6.277157e-07	0.000007	0.000010	50	10	0.000362	64	0.000066	ort	100
7	0.000008	9.626267e-07	0.000007	0.000010	10	10	0.000076	64	0.000066	ext	100
8	0.000007	9.653891e-08	0.000007	0.000007	10	10	0.000069	64	0.000015	ext_1_1	100
9	0.000082	3.342515e-05	0.000049	0.000171	50	50	0.004080	64	0.001063	ort	1000
10	0.000420	1.931256e-04	0.000264	0.001075	50	50	0.021020	64	0.014655	ext	1000
11	0.000328	8.118093e-05	0.000267	0.000612	50	50	0.016385	64	0.000313	ext_1_1	1000

Plots

piv = df.pivot(index="dim", columns="name", values="average")

fig, ax = plt.subplots(1, 1)
piv.plot(ax=ax, title="Binding Comparison", logy=True, logx=True)
fig.tight_layout()
fig.savefig("plot_bench_ort.png")