Note
Go to the end to download the full example code
TreeEnsemble, dense, and sparse¶
The example benchmarks the sparse implementation for TreeEnsemble. The default set of optimized parameters is very short and is meant to be executed fast. Many more parameters can be tried.
python plot_op_tree_ensemble_sparse --scenario=LONG
To change the training parameters:
python plot_op_tree_ensemble_sparse.py
--n_trees=100
--max_depth=10
--n_features=50
--sparsity=0.9
--batch_size=100000
Another example with a full list of parameters:
- python plot_op_tree_ensemble_sparse.py
–n_trees=100 –max_depth=10 –n_features=50 –batch_size=100000 –sparsity=0.9 –tries=3 –scenario=CUSTOM –parallel_tree=80,40 –parallel_tree_N=128,64 –parallel_N=50,25 –batch_size_tree=1,2 –batch_size_rows=1,2 –use_node3=0
Another example:
python plot_op_tree_ensemble_sparse.py
--n_trees=100 --n_features=10 --batch_size=10000 --max_depth=8 -s SHORT
import logging
import os
import timeit
from typing import Tuple
import numpy
import onnx
from onnx import ModelProto, TensorProto
from onnx.helper import make_graph, make_model, make_tensor_value_info
from pandas import DataFrame, concat
from sklearn.datasets import make_regression
from sklearn.ensemble import RandomForestRegressor
from skl2onnx import to_onnx
from onnxruntime import InferenceSession, SessionOptions
from onnx_array_api.plotting.text_plot import onnx_simple_text_plot
from onnx_extended.ortops.optim.cpu import get_ort_ext_libs
from onnx_extended.ortops.optim.optimize import (
change_onnx_operator_domain,
get_node_attribute,
optimize_model,
)
from onnx_extended.tools.onnx_nodes import multiply_tree
from onnx_extended.validation.cpu._validation import dense_to_sparse_struct
from onnx_extended.plotting.benchmark import hhistograms
from onnx_extended.args import get_parsed_args
from onnx_extended.ext_test_case import unit_test_going
logging.getLogger("matplotlib.font_manager").setLevel(logging.ERROR)
script_args = get_parsed_args(
"plot_op_tree_ensemble_sparse",
description=__doc__,
scenarios={
"SHORT": "short optimization (default)",
"LONG": "test more options",
"CUSTOM": "use values specified by the command line",
},
sparsity=(0.99, "input sparsity"),
n_features=(2 if unit_test_going() else 500, "number of features to generate"),
n_trees=(3 if unit_test_going() else 10, "number of trees to train"),
max_depth=(2 if unit_test_going() else 10, "max_depth"),
batch_size=(1000 if unit_test_going() else 1000, "batch size"),
parallel_tree=("80,160,40", "values to try for parallel_tree"),
parallel_tree_N=("256,128,64", "values to try for parallel_tree_N"),
parallel_N=("100,50,25", "values to try for parallel_N"),
batch_size_tree=("2,4,8", "values to try for batch_size_tree"),
batch_size_rows=("2,4,8", "values to try for batch_size_rows"),
use_node3=("0,1", "values to try for use_node3"),
expose="",
n_jobs=("-1", "number of jobs to train the RandomForestRegressor"),
)
Training a model¶
def train_model(
batch_size: int, n_features: int, n_trees: int, max_depth: int, sparsity: float
) -> Tuple[str, numpy.ndarray, numpy.ndarray]:
filename = (
f"plot_op_tree_ensemble_sparse-f{n_features}-{n_trees}-"
f"d{max_depth}-s{sparsity}.onnx"
)
if not os.path.exists(filename):
X, y = make_regression(
batch_size + max(batch_size, 2 ** (max_depth + 1)),
n_features=n_features,
n_targets=1,
)
mask = numpy.random.rand(*X.shape) <= sparsity
X[mask] = 0
X, y = X.astype(numpy.float32), y.astype(numpy.float32)
print(f"Training to get {filename!r} with X.shape={X.shape}")
# To be faster, we train only 1 tree.
model = RandomForestRegressor(
1, max_depth=max_depth, verbose=2, n_jobs=int(script_args.n_jobs)
)
model.fit(X[:-batch_size], y[:-batch_size])
onx = to_onnx(model, X[:1], target_opset={"": 18, "ai.onnx.ml": 3})
# And wd multiply the trees.
node = multiply_tree(onx.graph.node[0], n_trees)
onx = make_model(
make_graph([node], onx.graph.name, onx.graph.input, onx.graph.output),
domain=onx.domain,
opset_imports=onx.opset_import,
ir_version=onx.ir_version,
)
with open(filename, "wb") as f:
f.write(onx.SerializeToString())
else:
X, y = make_regression(batch_size, n_features=n_features, n_targets=1)
mask = numpy.random.rand(*X.shape) <= sparsity
X[mask] = 0
X, y = X.astype(numpy.float32), y.astype(numpy.float32)
Xb, yb = X[-batch_size:].copy(), y[-batch_size:].copy()
return filename, Xb, yb
def measure_sparsity(x):
f = x.flatten()
return float((f == 0).astype(numpy.int64).sum()) / float(x.size)
batch_size = script_args.batch_size
n_features = script_args.n_features
n_trees = script_args.n_trees
max_depth = script_args.max_depth
sparsity = script_args.sparsity
print(f"batch_size={batch_size}")
print(f"n_features={n_features}")
print(f"n_trees={n_trees}")
print(f"max_depth={max_depth}")
print(f"sparsity={sparsity}")
batch_size=1000
n_features=500
n_trees=10
max_depth=10
sparsity=0.99
training
Training to get 'plot_op_tree_ensemble_sparse-f500-10-d10-s0.99.onnx' with X.shape=(3048, 500)
[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 8 concurrent workers.
building tree 1 of 1
[Parallel(n_jobs=-1)]: Done 1 out of 1 | elapsed: 0.0s finished
Xb.shape=(1000, 500)
yb.shape=(1000,)
measured sparsity=0.989778
Rewrite the onnx file to use a different kernel¶
The custom kernel is mapped to a custom operator with the same name the attributes and domain = “onnx_extended.ortops.optim.cpu”. We call a function to do that replacement. First the current model.
opset: domain='ai.onnx.ml' version=1
opset: domain='' version=18
input: name='X' type=dtype('float32') shape=['', 500]
TreeEnsembleRegressor(X, n_targets=1, nodes_falsenodeids=570:[52,43,42...56,0,0], nodes_featureids=570:[48,430,220...191,298,35], nodes_hitrates=570:[1.0,1.0...1.0,1.0], nodes_missing_value_tracks_true=570:[0,0,0...0,0,0], nodes_modes=570:[b'BRANCH_LEQ',b'BRANCH_LEQ'...b'LEAF',b'LEAF'], nodes_nodeids=570:[0,1,2...54,55,56], nodes_treeids=570:[0,0,0...9,9,9], nodes_truenodeids=570:[1,2,3...55,0,0], nodes_values=570:[0.9763897061347961,1.0169752836227417...0.7212909460067749,-0.2563537657260895], post_transform=b'NONE', target_ids=290:[0,0,0...0,0,0], target_nodeids=290:[5,6,9...53,55,56], target_treeids=290:[0,0,0...9,9,9], target_weights=290:[406.1909484863281,240.94696044921875...336.6980895996094,361.9876708984375]) -> variable
output: name='variable' type=dtype('float32') shape=['', 1]
And then the modified model.
def transform_model(model, use_sparse=False, **kwargs):
onx = ModelProto()
onx.ParseFromString(model.SerializeToString())
att = get_node_attribute(onx.graph.node[0], "nodes_modes")
modes = ",".join(map(lambda s: s.decode("ascii"), att.strings)).replace(
"BRANCH_", ""
)
if use_sparse and "new_op_type" not in kwargs:
kwargs["new_op_type"] = "TreeEnsembleRegressorSparse"
if use_sparse:
# with sparse tensor, missing value means 0
att = get_node_attribute(onx.graph.node[0], "nodes_values")
thresholds = numpy.array(att.floats, dtype=numpy.float32)
missing_true = (thresholds >= 0).astype(numpy.int64)
kwargs["nodes_missing_value_tracks_true"] = missing_true
new_onx = change_onnx_operator_domain(
onx,
op_type="TreeEnsembleRegressor",
op_domain="ai.onnx.ml",
new_op_domain="onnx_extended.ortops.optim.cpu",
nodes_modes=modes,
**kwargs,
)
if use_sparse:
del new_onx.graph.input[:]
new_onx.graph.input.append(
make_tensor_value_info("X", TensorProto.FLOAT, (None,))
)
return new_onx
print("Tranform model to add a custom node.")
onx_modified = transform_model(onx)
print(f"Save into {filename + 'modified.onnx'!r}.")
with open(filename + "modified.onnx", "wb") as f:
f.write(onx_modified.SerializeToString())
print("done.")
print(onnx_simple_text_plot(onx_modified))
Tranform model to add a custom node.
Save into 'plot_op_tree_ensemble_sparse-f500-10-d10-s0.99.onnxmodified.onnx'.
done.
opset: domain='ai.onnx.ml' version=1
opset: domain='' version=18
opset: domain='onnx_extended.ortops.optim.cpu' version=1
input: name='X' type=dtype('float32') shape=['', 500]
TreeEnsembleRegressor[onnx_extended.ortops.optim.cpu](X, nodes_modes=b'LEQ,LEQ,LEQ,LEQ,LEQ,LEAF,LEAF,LEQ,LEQ,...LEAF,LEAF', n_targets=1, nodes_falsenodeids=570:[52,43,42...56,0,0], nodes_featureids=570:[48,430,220...191,298,35], nodes_hitrates=570:[1.0,1.0...1.0,1.0], nodes_missing_value_tracks_true=570:[0,0,0...0,0,0], nodes_nodeids=570:[0,1,2...54,55,56], nodes_treeids=570:[0,0,0...9,9,9], nodes_truenodeids=570:[1,2,3...55,0,0], nodes_values=570:[0.9763897061347961,1.0169752836227417...0.7212909460067749,-0.2563537657260895], post_transform=b'NONE', target_ids=290:[0,0,0...0,0,0], target_nodeids=290:[5,6,9...53,55,56], target_treeids=290:[0,0,0...9,9,9], target_weights=290:[406.1909484863281,240.94696044921875...336.6980895996094,361.9876708984375]) -> variable
output: name='variable' type=dtype('float32') shape=['', 1]
Same with sparse.
print("Same transformation but with sparse.")
onx_modified_sparse = transform_model(onx, use_sparse=True)
print(f"Save into {filename + 'modified.sparse.onnx'!r}.")
with open(filename + "modified.sparse.onnx", "wb") as f:
f.write(onx_modified_sparse.SerializeToString())
print("done.")
print(onnx_simple_text_plot(onx_modified_sparse))
Same transformation but with sparse.
Save into 'plot_op_tree_ensemble_sparse-f500-10-d10-s0.99.onnxmodified.sparse.onnx'.
done.
opset: domain='ai.onnx.ml' version=1
opset: domain='' version=18
opset: domain='onnx_extended.ortops.optim.cpu' version=1
input: name='X' type=dtype('float32') shape=['']
TreeEnsembleRegressorSparse[onnx_extended.ortops.optim.cpu](X, nodes_missing_value_tracks_true=570:[1,1,1...0,1,0], nodes_modes=b'LEQ,LEQ,LEQ,LEQ,LEQ,LEAF,LEAF,LEQ,LEQ,...LEAF,LEAF', n_targets=1, nodes_falsenodeids=570:[52,43,42...56,0,0], nodes_featureids=570:[48,430,220...191,298,35], nodes_hitrates=570:[1.0,1.0...1.0,1.0], nodes_nodeids=570:[0,1,2...54,55,56], nodes_treeids=570:[0,0,0...9,9,9], nodes_truenodeids=570:[1,2,3...55,0,0], nodes_values=570:[0.9763897061347961,1.0169752836227417...0.7212909460067749,-0.2563537657260895], post_transform=b'NONE', target_ids=290:[0,0,0...0,0,0], target_nodeids=290:[5,6,9...53,55,56], target_treeids=290:[0,0,0...9,9,9], target_weights=290:[406.1909484863281,240.94696044921875...336.6980895996094,361.9876708984375]) -> variable
output: name='variable' type=dtype('float32') shape=['', 1]
Comparing onnxruntime and the custom kernel¶
print(f"Loading {filename!r}")
sess_ort = InferenceSession(filename, providers=["CPUExecutionProvider"])
r = get_ort_ext_libs()
print(f"Creating SessionOptions with {r!r}")
opts = SessionOptions()
if r is not None:
opts.register_custom_ops_library(r[0])
print(f"Loading modified {filename!r}")
sess_cus = InferenceSession(
onx_modified.SerializeToString(), opts, providers=["CPUExecutionProvider"]
)
print(f"Loading modified sparse {filename!r}")
sess_cus_sparse = InferenceSession(
onx_modified_sparse.SerializeToString(), opts, providers=["CPUExecutionProvider"]
)
print(f"Running once with shape {Xb.shape}.")
base = sess_ort.run(None, {"X": Xb})[0]
print(f"Running modified with shape {Xb.shape}.")
got = sess_cus.run(None, {"X": Xb})[0]
print("done.")
Xb_sp = dense_to_sparse_struct(Xb)
print(f"Running modified sparse with shape {Xb_sp.shape}.")
got_sparse = sess_cus_sparse.run(None, {"X": Xb_sp})[0]
print("done.")
Loading 'plot_op_tree_ensemble_sparse-f500-10-d10-s0.99.onnx'
Creating SessionOptions with ['/home/xadupre/github/onnx-extended/onnx_extended/ortops/optim/cpu/libortops_optim_cpu.so']
Loading modified 'plot_op_tree_ensemble_sparse-f500-10-d10-s0.99.onnx'
Loading modified sparse 'plot_op_tree_ensemble_sparse-f500-10-d10-s0.99.onnx'
Running once with shape (1000, 500).
Running modified with shape (1000, 500).
done.
Running modified sparse with shape (10278,).
done.
Discrepancies?
Discrepancies: 0.000244140625
Discrepancies sparse: 0.000244140625
Simple verification¶
Baseline with onnxruntime.
t1 = timeit.timeit(lambda: sess_ort.run(None, {"X": Xb}), number=50)
print(f"baseline: {t1}")
baseline: 0.027769100001023617
The custom implementation.
t2 = timeit.timeit(lambda: sess_cus.run(None, {"X": Xb}), number=50)
print(f"new time: {t2}")
new time: 0.0045196999999461696
The custom sparse implementation.
t3 = timeit.timeit(lambda: sess_cus_sparse.run(None, {"X": Xb_sp}), number=50)
print(f"new time sparse: {t3}")
new time sparse: 0.01739589999851887
Time for comparison¶
The custom kernel supports the same attributes as TreeEnsembleRegressor plus new ones to tune the parallelization. They can be seen in tree_ensemble.cc. Let’s try out many possibilities. The default values are the first ones.
if unit_test_going():
optim_params = dict(
parallel_tree=[40], # default is 80
parallel_tree_N=[128], # default is 128
parallel_N=[50, 25], # default is 50
batch_size_tree=[1], # default is 1
batch_size_rows=[1], # default is 1
use_node3=[0], # default is 0
)
elif script_args.scenario in (None, "SHORT"):
optim_params = dict(
parallel_tree=[80, 40], # default is 80
parallel_tree_N=[128, 64], # default is 128
parallel_N=[50, 25], # default is 50
batch_size_tree=[1], # default is 1
batch_size_rows=[1], # default is 1
use_node3=[0], # default is 0
)
elif script_args.scenario == "LONG":
optim_params = dict(
parallel_tree=[80, 160, 40],
parallel_tree_N=[256, 128, 64],
parallel_N=[100, 50, 25],
batch_size_tree=[1, 2, 4, 8],
batch_size_rows=[1, 2, 4, 8],
use_node3=[0, 1],
)
elif script_args.scenario == "CUSTOM":
optim_params = dict(
parallel_tree=list(int(i) for i in script_args.parallel_tree.split(",")),
parallel_tree_N=list(int(i) for i in script_args.parallel_tree_N.split(",")),
parallel_N=list(int(i) for i in script_args.parallel_N.split(",")),
batch_size_tree=list(int(i) for i in script_args.batch_size_tree.split(",")),
batch_size_rows=list(int(i) for i in script_args.batch_size_rows.split(",")),
use_node3=list(int(i) for i in script_args.use_node3.split(",")),
)
else:
raise ValueError(
f"Unknown scenario {script_args.scenario!r}, use --help to get them."
)
cmds = []
for att, value in optim_params.items():
cmds.append(f"--{att}={','.join(map(str, value))}")
print("Full list of optimization parameters:")
print(" ".join(cmds))
Full list of optimization parameters:
--parallel_tree=80,40 --parallel_tree_N=128,64 --parallel_N=50,25 --batch_size_tree=1 --batch_size_rows=1 --use_node3=0
Then the optimization for dense
def create_session(onx):
opts = SessionOptions()
r = get_ort_ext_libs()
if r is None:
raise RuntimeError("No custom implementation available.")
opts.register_custom_ops_library(r[0])
return InferenceSession(
onx.SerializeToString(), opts, providers=["CPUExecutionProvider"]
)
res = optimize_model(
onx,
feeds={"X": Xb},
transform=transform_model,
session=create_session,
baseline=lambda onx: InferenceSession(
onx.SerializeToString(), providers=["CPUExecutionProvider"]
),
params=optim_params,
verbose=True,
number=script_args.number,
repeat=script_args.repeat,
warmup=script_args.warmup,
sleep=script_args.sleep,
n_tries=script_args.tries,
)
0%| | 0/16 [00:00<?, ?it/s]
i=1/16 TRY=0 //tree=80 //tree_N=128 //N=50 bs_tree=1 batch_size_rows=1 n3=0: 0%| | 0/16 [00:00<?, ?it/s]
i=1/16 TRY=0 //tree=80 //tree_N=128 //N=50 bs_tree=1 batch_size_rows=1 n3=0: 6%|▋ | 1/16 [00:00<00:05, 2.93it/s]
i=2/16 TRY=0 //tree=80 //tree_N=128 //N=25 bs_tree=1 batch_size_rows=1 n3=0 ~=2.36x: 6%|▋ | 1/16 [00:00<00:05, 2.93it/s]
i=2/16 TRY=0 //tree=80 //tree_N=128 //N=25 bs_tree=1 batch_size_rows=1 n3=0 ~=2.36x: 12%|█▎ | 2/16 [00:00<00:03, 4.35it/s]
i=3/16 TRY=0 //tree=80 //tree_N=64 //N=50 bs_tree=1 batch_size_rows=1 n3=0 ~=2.36x: 12%|█▎ | 2/16 [00:00<00:03, 4.35it/s]
i=3/16 TRY=0 //tree=80 //tree_N=64 //N=50 bs_tree=1 batch_size_rows=1 n3=0 ~=2.36x: 19%|█▉ | 3/16 [00:00<00:02, 5.21it/s]
i=4/16 TRY=0 //tree=80 //tree_N=64 //N=25 bs_tree=1 batch_size_rows=1 n3=0 ~=2.66x: 19%|█▉ | 3/16 [00:00<00:02, 5.21it/s]
i=4/16 TRY=0 //tree=80 //tree_N=64 //N=25 bs_tree=1 batch_size_rows=1 n3=0 ~=2.66x: 25%|██▌ | 4/16 [00:00<00:02, 5.96it/s]
i=5/16 TRY=0 //tree=40 //tree_N=128 //N=50 bs_tree=1 batch_size_rows=1 n3=0 ~=4.82x: 25%|██▌ | 4/16 [00:00<00:02, 5.96it/s]
i=5/16 TRY=0 //tree=40 //tree_N=128 //N=50 bs_tree=1 batch_size_rows=1 n3=0 ~=4.82x: 31%|███▏ | 5/16 [00:00<00:01, 6.16it/s]
i=6/16 TRY=0 //tree=40 //tree_N=128 //N=25 bs_tree=1 batch_size_rows=1 n3=0 ~=4.82x: 31%|███▏ | 5/16 [00:00<00:01, 6.16it/s]
i=6/16 TRY=0 //tree=40 //tree_N=128 //N=25 bs_tree=1 batch_size_rows=1 n3=0 ~=4.82x: 38%|███▊ | 6/16 [00:01<00:01, 5.94it/s]
i=7/16 TRY=0 //tree=40 //tree_N=64 //N=50 bs_tree=1 batch_size_rows=1 n3=0 ~=4.82x: 38%|███▊ | 6/16 [00:01<00:01, 5.94it/s]
i=7/16 TRY=0 //tree=40 //tree_N=64 //N=50 bs_tree=1 batch_size_rows=1 n3=0 ~=4.82x: 44%|████▍ | 7/16 [00:01<00:01, 6.10it/s]
i=8/16 TRY=0 //tree=40 //tree_N=64 //N=25 bs_tree=1 batch_size_rows=1 n3=0 ~=4.82x: 44%|████▍ | 7/16 [00:01<00:01, 6.10it/s]
i=8/16 TRY=0 //tree=40 //tree_N=64 //N=25 bs_tree=1 batch_size_rows=1 n3=0 ~=4.82x: 50%|█████ | 8/16 [00:01<00:01, 6.16it/s]
i=9/16 TRY=1 //tree=80 //tree_N=128 //N=50 bs_tree=1 batch_size_rows=1 n3=0 ~=4.82x: 50%|█████ | 8/16 [00:01<00:01, 6.16it/s]
i=9/16 TRY=1 //tree=80 //tree_N=128 //N=50 bs_tree=1 batch_size_rows=1 n3=0 ~=4.82x: 56%|█████▋ | 9/16 [00:01<00:01, 6.28it/s]
i=10/16 TRY=1 //tree=80 //tree_N=128 //N=25 bs_tree=1 batch_size_rows=1 n3=0 ~=4.82x: 56%|█████▋ | 9/16 [00:01<00:01, 6.28it/s]
i=10/16 TRY=1 //tree=80 //tree_N=128 //N=25 bs_tree=1 batch_size_rows=1 n3=0 ~=4.82x: 62%|██████▎ | 10/16 [00:01<00:00, 6.13it/s]
i=11/16 TRY=1 //tree=80 //tree_N=64 //N=50 bs_tree=1 batch_size_rows=1 n3=0 ~=4.82x: 62%|██████▎ | 10/16 [00:01<00:00, 6.13it/s]
i=11/16 TRY=1 //tree=80 //tree_N=64 //N=50 bs_tree=1 batch_size_rows=1 n3=0 ~=4.82x: 69%|██████▉ | 11/16 [00:01<00:00, 6.19it/s]
i=12/16 TRY=1 //tree=80 //tree_N=64 //N=25 bs_tree=1 batch_size_rows=1 n3=0 ~=4.82x: 69%|██████▉ | 11/16 [00:01<00:00, 6.19it/s]
i=12/16 TRY=1 //tree=80 //tree_N=64 //N=25 bs_tree=1 batch_size_rows=1 n3=0 ~=4.82x: 75%|███████▌ | 12/16 [00:02<00:00, 6.21it/s]
i=13/16 TRY=1 //tree=40 //tree_N=128 //N=50 bs_tree=1 batch_size_rows=1 n3=0 ~=4.82x: 75%|███████▌ | 12/16 [00:02<00:00, 6.21it/s]
i=13/16 TRY=1 //tree=40 //tree_N=128 //N=50 bs_tree=1 batch_size_rows=1 n3=0 ~=4.82x: 81%|████████▏ | 13/16 [00:02<00:00, 6.16it/s]
i=14/16 TRY=1 //tree=40 //tree_N=128 //N=25 bs_tree=1 batch_size_rows=1 n3=0 ~=4.82x: 81%|████████▏ | 13/16 [00:02<00:00, 6.16it/s]
i=14/16 TRY=1 //tree=40 //tree_N=128 //N=25 bs_tree=1 batch_size_rows=1 n3=0 ~=4.82x: 88%|████████▊ | 14/16 [00:02<00:00, 6.08it/s]
i=15/16 TRY=1 //tree=40 //tree_N=64 //N=50 bs_tree=1 batch_size_rows=1 n3=0 ~=4.82x: 88%|████████▊ | 14/16 [00:02<00:00, 6.08it/s]
i=15/16 TRY=1 //tree=40 //tree_N=64 //N=50 bs_tree=1 batch_size_rows=1 n3=0 ~=4.82x: 94%|█████████▍| 15/16 [00:02<00:00, 6.12it/s]
i=16/16 TRY=1 //tree=40 //tree_N=64 //N=25 bs_tree=1 batch_size_rows=1 n3=0 ~=4.82x: 94%|█████████▍| 15/16 [00:02<00:00, 6.12it/s]
i=16/16 TRY=1 //tree=40 //tree_N=64 //N=25 bs_tree=1 batch_size_rows=1 n3=0 ~=4.82x: 100%|██████████| 16/16 [00:02<00:00, 6.24it/s]
i=16/16 TRY=1 //tree=40 //tree_N=64 //N=25 bs_tree=1 batch_size_rows=1 n3=0 ~=4.82x: 100%|██████████| 16/16 [00:02<00:00, 5.90it/s]
Then the optimization for sparse
res_sparse = optimize_model(
onx,
feeds={"X": Xb_sp},
transform=lambda *args, **kwargs: transform_model(*args, use_sparse=True, **kwargs),
session=create_session,
params=optim_params,
verbose=True,
number=script_args.number,
repeat=script_args.repeat,
warmup=script_args.warmup,
sleep=script_args.sleep,
n_tries=script_args.tries,
)
0%| | 0/16 [00:00<?, ?it/s]
i=1/16 TRY=0 //tree=80 //tree_N=128 //N=50 bs_tree=1 batch_size_rows=1 n3=0: 0%| | 0/16 [00:00<?, ?it/s]
i=1/16 TRY=0 //tree=80 //tree_N=128 //N=50 bs_tree=1 batch_size_rows=1 n3=0: 6%|▋ | 1/16 [00:00<00:03, 4.00it/s]
i=2/16 TRY=0 //tree=80 //tree_N=128 //N=25 bs_tree=1 batch_size_rows=1 n3=0: 6%|▋ | 1/16 [00:00<00:03, 4.00it/s]
i=2/16 TRY=0 //tree=80 //tree_N=128 //N=25 bs_tree=1 batch_size_rows=1 n3=0: 12%|█▎ | 2/16 [00:00<00:02, 5.14it/s]
i=3/16 TRY=0 //tree=80 //tree_N=64 //N=50 bs_tree=1 batch_size_rows=1 n3=0: 12%|█▎ | 2/16 [00:00<00:02, 5.14it/s]
i=3/16 TRY=0 //tree=80 //tree_N=64 //N=50 bs_tree=1 batch_size_rows=1 n3=0: 19%|█▉ | 3/16 [00:00<00:02, 4.91it/s]
i=4/16 TRY=0 //tree=80 //tree_N=64 //N=25 bs_tree=1 batch_size_rows=1 n3=0: 19%|█▉ | 3/16 [00:00<00:02, 4.91it/s]
i=4/16 TRY=0 //tree=80 //tree_N=64 //N=25 bs_tree=1 batch_size_rows=1 n3=0: 25%|██▌ | 4/16 [00:00<00:02, 4.64it/s]
i=5/16 TRY=0 //tree=40 //tree_N=128 //N=50 bs_tree=1 batch_size_rows=1 n3=0: 25%|██▌ | 4/16 [00:00<00:02, 4.64it/s]
i=5/16 TRY=0 //tree=40 //tree_N=128 //N=50 bs_tree=1 batch_size_rows=1 n3=0: 31%|███▏ | 5/16 [00:01<00:02, 4.89it/s]
i=6/16 TRY=0 //tree=40 //tree_N=128 //N=25 bs_tree=1 batch_size_rows=1 n3=0: 31%|███▏ | 5/16 [00:01<00:02, 4.89it/s]
i=6/16 TRY=0 //tree=40 //tree_N=128 //N=25 bs_tree=1 batch_size_rows=1 n3=0: 38%|███▊ | 6/16 [00:01<00:02, 4.44it/s]
i=7/16 TRY=0 //tree=40 //tree_N=64 //N=50 bs_tree=1 batch_size_rows=1 n3=0: 38%|███▊ | 6/16 [00:01<00:02, 4.44it/s]
i=7/16 TRY=0 //tree=40 //tree_N=64 //N=50 bs_tree=1 batch_size_rows=1 n3=0: 44%|████▍ | 7/16 [00:01<00:02, 4.33it/s]
i=8/16 TRY=0 //tree=40 //tree_N=64 //N=25 bs_tree=1 batch_size_rows=1 n3=0: 44%|████▍ | 7/16 [00:01<00:02, 4.33it/s]
i=8/16 TRY=0 //tree=40 //tree_N=64 //N=25 bs_tree=1 batch_size_rows=1 n3=0: 50%|█████ | 8/16 [00:01<00:01, 4.57it/s]
i=9/16 TRY=1 //tree=80 //tree_N=128 //N=50 bs_tree=1 batch_size_rows=1 n3=0: 50%|█████ | 8/16 [00:01<00:01, 4.57it/s]
i=9/16 TRY=1 //tree=80 //tree_N=128 //N=50 bs_tree=1 batch_size_rows=1 n3=0: 56%|█████▋ | 9/16 [00:02<00:01, 4.24it/s]
i=10/16 TRY=1 //tree=80 //tree_N=128 //N=25 bs_tree=1 batch_size_rows=1 n3=0: 56%|█████▋ | 9/16 [00:02<00:01, 4.24it/s]
i=10/16 TRY=1 //tree=80 //tree_N=128 //N=25 bs_tree=1 batch_size_rows=1 n3=0: 62%|██████▎ | 10/16 [00:02<00:01, 4.30it/s]
i=11/16 TRY=1 //tree=80 //tree_N=64 //N=50 bs_tree=1 batch_size_rows=1 n3=0: 62%|██████▎ | 10/16 [00:02<00:01, 4.30it/s]
i=11/16 TRY=1 //tree=80 //tree_N=64 //N=50 bs_tree=1 batch_size_rows=1 n3=0: 69%|██████▉ | 11/16 [00:02<00:01, 4.15it/s]
i=12/16 TRY=1 //tree=80 //tree_N=64 //N=25 bs_tree=1 batch_size_rows=1 n3=0: 69%|██████▉ | 11/16 [00:02<00:01, 4.15it/s]
i=12/16 TRY=1 //tree=80 //tree_N=64 //N=25 bs_tree=1 batch_size_rows=1 n3=0: 75%|███████▌ | 12/16 [00:02<00:00, 4.49it/s]
i=13/16 TRY=1 //tree=40 //tree_N=128 //N=50 bs_tree=1 batch_size_rows=1 n3=0: 75%|███████▌ | 12/16 [00:02<00:00, 4.49it/s]
i=13/16 TRY=1 //tree=40 //tree_N=128 //N=50 bs_tree=1 batch_size_rows=1 n3=0: 81%|████████▏ | 13/16 [00:02<00:00, 4.48it/s]
i=14/16 TRY=1 //tree=40 //tree_N=128 //N=25 bs_tree=1 batch_size_rows=1 n3=0: 81%|████████▏ | 13/16 [00:02<00:00, 4.48it/s]
i=14/16 TRY=1 //tree=40 //tree_N=128 //N=25 bs_tree=1 batch_size_rows=1 n3=0: 88%|████████▊ | 14/16 [00:03<00:00, 4.39it/s]
i=15/16 TRY=1 //tree=40 //tree_N=64 //N=50 bs_tree=1 batch_size_rows=1 n3=0: 88%|████████▊ | 14/16 [00:03<00:00, 4.39it/s]
i=15/16 TRY=1 //tree=40 //tree_N=64 //N=50 bs_tree=1 batch_size_rows=1 n3=0: 94%|█████████▍| 15/16 [00:03<00:00, 4.40it/s]
i=16/16 TRY=1 //tree=40 //tree_N=64 //N=25 bs_tree=1 batch_size_rows=1 n3=0: 94%|█████████▍| 15/16 [00:03<00:00, 4.40it/s]
i=16/16 TRY=1 //tree=40 //tree_N=64 //N=25 bs_tree=1 batch_size_rows=1 n3=0: 100%|██████████| 16/16 [00:03<00:00, 4.47it/s]
i=16/16 TRY=1 //tree=40 //tree_N=64 //N=25 bs_tree=1 batch_size_rows=1 n3=0: 100%|██████████| 16/16 [00:03<00:00, 4.47it/s]
And the results.
df_dense = DataFrame(res)
df_dense["input"] = "dense"
df_sparse = DataFrame(res_sparse)
df_sparse["input"] = "sparse"
df = concat([df_dense, df_sparse], axis=0)
df.to_csv("plot_op_tree_ensemble_sparse.csv", index=False)
df.to_excel("plot_op_tree_ensemble_sparse.xlsx", index=False)
print(df.columns)
print(df.head(5))
Index(['average', 'deviation', 'min_exec', 'max_exec', 'repeat', 'number',
'ttime', 'context_size', 'warmup_time', 'n_exp', 'n_exp_name',
'short_name', 'TRY', 'name', 'parallel_tree', 'parallel_tree_N',
'parallel_N', 'batch_size_tree', 'batch_size_rows', 'use_node3',
'input'],
dtype='object')
average deviation min_exec max_exec repeat number ttime ... parallel_tree parallel_tree_N parallel_N batch_size_tree batch_size_rows use_node3 input
0 0.000817 0.000345 0.000500 0.001617 10 10 0.008168 ... NaN NaN NaN NaN NaN NaN dense
1 0.000347 0.000136 0.000266 0.000743 10 10 0.003465 ... 80.0 128.0 50.0 1.0 1.0 0.0 dense
2 0.000352 0.000108 0.000249 0.000623 10 10 0.003515 ... 80.0 128.0 25.0 1.0 1.0 0.0 dense
3 0.000307 0.000134 0.000200 0.000680 10 10 0.003066 ... 80.0 64.0 50.0 1.0 1.0 0.0 dense
4 0.000169 0.000034 0.000134 0.000228 10 10 0.001694 ... 80.0 64.0 25.0 1.0 1.0 0.0 dense
[5 rows x 21 columns]
Sorting¶
small_df = df.drop(
[
"min_exec",
"max_exec",
"repeat",
"number",
"context_size",
"n_exp_name",
],
axis=1,
).sort_values("average")
print(small_df.head(n=10))
average deviation ttime warmup_time n_exp short_name TRY ... parallel_tree parallel_tree_N parallel_N batch_size_tree batch_size_rows use_node3 input
4 0.000169 0.000034 0.001694 0.001401 3 0,80,64,25,1,1,0 NaN ... 80.0 64.0 25.0 1.0 1.0 0.0 dense
3 0.000307 0.000134 0.003066 0.002149 2 0,80,64,50,1,1,0 NaN ... 80.0 64.0 50.0 1.0 1.0 0.0 dense
1 0.000347 0.000136 0.003465 0.002939 0 0,80,128,50,1,1,0 NaN ... 80.0 128.0 50.0 1.0 1.0 0.0 dense
2 0.000352 0.000108 0.003515 0.002619 1 0,80,128,25,1,1,0 NaN ... 80.0 128.0 25.0 1.0 1.0 0.0 dense
9 0.000355 0.000133 0.003553 0.002988 8 1,80,128,50,1,1,0 NaN ... 80.0 128.0 50.0 1.0 1.0 0.0 dense
16 0.000357 0.000044 0.003568 0.003135 15 1,40,64,25,1,1,0 NaN ... 40.0 64.0 25.0 1.0 1.0 0.0 dense
11 0.000358 0.000151 0.003585 0.002161 10 1,80,64,50,1,1,0 NaN ... 80.0 64.0 50.0 1.0 1.0 0.0 dense
7 0.000367 0.000144 0.003669 0.002629 6 0,40,64,50,1,1,0 NaN ... 40.0 64.0 50.0 1.0 1.0 0.0 dense
5 0.000402 0.000211 0.004019 0.002052 4 0,40,128,50,1,1,0 NaN ... 40.0 128.0 50.0 1.0 1.0 0.0 dense
12 0.000408 0.000236 0.004082 0.002328 11 1,80,64,25,1,1,0 NaN ... 80.0 64.0 25.0 1.0 1.0 0.0 dense
[10 rows x 15 columns]
Worst¶
print(small_df.tail(n=10))
average deviation ttime warmup_time n_exp short_name TRY ... parallel_tree parallel_tree_N parallel_N batch_size_tree batch_size_rows use_node3 input
12 0.001087 0.000513 0.010873 0.007869 12 1,40,128,50,1,1,0 NaN ... 40.0 128.0 50.0 1.0 1.0 0.0 sparse
9 0.001114 0.000487 0.011142 0.008037 9 1,80,128,25,1,1,0 NaN ... 80.0 128.0 25.0 1.0 1.0 0.0 sparse
3 0.001115 0.000793 0.011146 0.013622 3 0,80,64,25,1,1,0 NaN ... 80.0 64.0 25.0 1.0 1.0 0.0 sparse
14 0.001134 0.000566 0.011341 0.006392 14 1,40,64,50,1,1,0 NaN ... 40.0 64.0 50.0 1.0 1.0 0.0 sparse
13 0.001227 0.000921 0.012271 0.007970 13 1,40,128,25,1,1,0 NaN ... 40.0 128.0 25.0 1.0 1.0 0.0 sparse
6 0.001346 0.000780 0.013464 0.002560 6 0,40,64,50,1,1,0 NaN ... 40.0 64.0 50.0 1.0 1.0 0.0 sparse
0 0.001347 0.000541 0.013466 0.007364 0 0,80,128,50,1,1,0 NaN ... 80.0 128.0 50.0 1.0 1.0 0.0 sparse
10 0.001439 0.001463 0.014393 0.009754 10 1,80,64,50,1,1,0 NaN ... 80.0 64.0 50.0 1.0 1.0 0.0 sparse
5 0.001482 0.000938 0.014823 0.011473 5 0,40,128,25,1,1,0 NaN ... 40.0 128.0 25.0 1.0 1.0 0.0 sparse
8 0.001603 0.002465 0.016026 0.005522 8 1,80,128,50,1,1,0 NaN ... 80.0 128.0 50.0 1.0 1.0 0.0 sparse
[10 rows x 15 columns]
Plot¶
skeys = ",".join(optim_params.keys())
title = f"TreeEnsemble tuning, n_tries={script_args.tries}\n{skeys}\nlower is better"
ax = hhistograms(df, title=title, keys=("input", "name"))
fig = ax.get_figure()
fig.savefig("plot_op_tree_ensemble_sparse.png")
Total running time of the script: (0 minutes 7.315 seconds)