Compares filtering implementations (numpy, cython)

The benchmark looks into different ways to implement thresholding: every value of a vector superior to mx is replaced by mx (numpy.clip()). It compares several implementation to numpy.

• cfilter_dmax cfilter_dmax

• cfilter_dmax2 cfilter_dmax2

• cfilter_dmax4 cfilter_dmax4

• cfilter_dmax16 cfilter_dmax16

• cyfilter_dmax cyfilter_dmax

• filter_dmax_cython filter_dmax_cython

• filter_dmax_cython_optim filter_dmax_cython_optim

• pyfilter_dmax pyfilter_dmax

import pprint
import numpy
import matplotlib.pyplot as plt
from pandas import DataFrame
from teachcompute.validation.cython.experiment_cython import (
    pyfilter_dmax,
    filter_dmax_cython,
    filter_dmax_cython_optim,
    cyfilter_dmax,
    cfilter_dmax,
    cfilter_dmax2,
    cfilter_dmax16,
    cfilter_dmax4,
)
from teachcompute.ext_test_case import measure_time_dim


def get_vectors(fct, n, h=200, dtype=numpy.float64):
    ctxs = [
        dict(
            va=numpy.random.randn(n).astype(dtype),
            fil=fct,
            mx=numpy.float64(0),
            x_name=n,
        )
        for n in range(10, n, h)
    ]
    return ctxs


def numpy_filter(va, mx):
    va[va > mx] = mx


all_res = []
for fct in [
    numpy_filter,
    pyfilter_dmax,
    filter_dmax_cython,
    filter_dmax_cython_optim,
    cyfilter_dmax,
    cfilter_dmax,
    cfilter_dmax2,
    cfilter_dmax16,
    cfilter_dmax4,
]:
    print(fct)
    ctxs = get_vectors(fct, 1000 if fct == pyfilter_dmax else 40000)
    res = list(measure_time_dim("fil(va, mx)", ctxs, verbose=1))
    for r in res:
        r["fct"] = fct.__name__
    all_res.extend(res)

pprint.pprint(all_res[:2])

<function numpy_filter at 0x7f77de7f60e0>

  0%|          | 0/200 [00:00<?, ?it/s]
 32%|███▎      | 65/200 [00:00<00:00, 642.06it/s]
 65%|██████▌   | 130/200 [00:00<00:00, 352.29it/s]
 86%|████████▋ | 173/200 [00:00<00:00, 290.31it/s]
100%|██████████| 200/200 [00:00<00:00, 287.11it/s]
<cyfunction pyfilter_dmax at 0x7f78147f4040>

  0%|          | 0/5 [00:00<?, ?it/s]
100%|██████████| 5/5 [00:00<00:00, 47.40it/s]
100%|██████████| 5/5 [00:00<00:00, 47.23it/s]
<cyfunction filter_dmax_cython at 0x7f77df267100>

  0%|          | 0/200 [00:00<?, ?it/s]
 40%|████      | 80/200 [00:00<00:00, 787.05it/s]
 80%|███████▉  | 159/200 [00:00<00:00, 448.56it/s]
100%|██████████| 200/200 [00:00<00:00, 381.32it/s]
<cyfunction filter_dmax_cython_optim at 0x7f77df267030>

  0%|          | 0/200 [00:00<?, ?it/s]
 42%|████▏     | 83/200 [00:00<00:00, 824.14it/s]
 83%|████████▎ | 166/200 [00:00<00:00, 462.81it/s]
100%|██████████| 200/200 [00:00<00:00, 345.57it/s]
<cyfunction cyfilter_dmax at 0x7f77df2668e0>

  0%|          | 0/200 [00:00<?, ?it/s]
 42%|████▏     | 83/200 [00:00<00:00, 827.69it/s]
 83%|████████▎ | 166/200 [00:00<00:00, 392.17it/s]
100%|██████████| 200/200 [00:00<00:00, 368.19it/s]
<cyfunction cfilter_dmax at 0x7f77df266a80>

  0%|          | 0/200 [00:00<?, ?it/s]
 41%|████      | 82/200 [00:00<00:00, 806.39it/s]
 82%|████████▏ | 163/200 [00:00<00:00, 431.72it/s]
100%|██████████| 200/200 [00:00<00:00, 407.29it/s]
<cyfunction cfilter_dmax2 at 0x7f77a87a4790>

  0%|          | 0/200 [00:00<?, ?it/s]
 42%|████▏     | 83/200 [00:00<00:00, 822.77it/s]
 83%|████████▎ | 166/200 [00:00<00:00, 403.14it/s]
100%|██████████| 200/200 [00:00<00:00, 344.72it/s]
<cyfunction cfilter_dmax16 at 0x7f77a87a46c0>

  0%|          | 0/200 [00:00<?, ?it/s]
 31%|███       | 62/200 [00:00<00:00, 612.79it/s]
 62%|██████▏   | 124/200 [00:00<00:00, 303.92it/s]
 82%|████████▏ | 163/200 [00:00<00:00, 228.71it/s]
 96%|█████████▌| 192/200 [00:00<00:00, 183.58it/s]
100%|██████████| 200/200 [00:00<00:00, 211.23it/s]
<cyfunction cfilter_dmax4 at 0x7f77a87a45f0>

  0%|          | 0/200 [00:00<?, ?it/s]
 24%|██▎       | 47/200 [00:00<00:00, 462.38it/s]
 47%|████▋     | 94/200 [00:00<00:00, 186.30it/s]
 60%|██████    | 121/200 [00:00<00:00, 139.86it/s]
 70%|███████   | 140/200 [00:00<00:00, 119.83it/s]
 78%|███████▊  | 155/200 [00:01<00:00, 103.18it/s]
 84%|████████▎ | 167/200 [00:01<00:00, 86.03it/s]
 88%|████████▊ | 177/200 [00:01<00:00, 82.16it/s]
 93%|█████████▎| 186/200 [00:01<00:00, 77.00it/s]
 97%|█████████▋| 194/200 [00:01<00:00, 72.78it/s]
100%|██████████| 200/200 [00:01<00:00, 102.08it/s]
[{'average': np.float64(1.0367240029154347e-06),
  'context_size': 232,
  'deviation': np.float64(1.4424319096952294e-07),
  'fct': 'numpy_filter',
  'max_exec': np.float64(1.4461799582932144e-06),
  'min_exec': np.float64(9.693200991023332e-07),
  'number': 50,
  'repeat': 10,
  'ttime': np.float64(1.0367240029154346e-05),
  'warmup_time': 5.065200093667954e-05,
  'x_name': 10},
 {'average': np.float64(1.0164739942410959e-06),
  'context_size': 232,
  'deviation': np.float64(1.0818667981810718e-08),
  'fct': 'numpy_filter',
  'max_exec': np.float64(1.045179960783571e-06),
  'min_exec': np.float64(1.0053000005427748e-06),
  'number': 50,
  'repeat': 10,
  'ttime': np.float64(1.0164739942410959e-05),
  'warmup_time': 8.527997124474496e-06,
  'x_name': 210}]

Let’s display the results

cc = DataFrame(all_res)
cc["N"] = cc["x_name"]

fig, ax = plt.subplots(2, 2, figsize=(10, 10))
cc[cc.N <= 1100].pivot(index="N", columns="fct", values="average").plot(
    logy=True, ax=ax[0, 0]
)
cc[cc.fct != "pyfilter_dmax"].pivot(index="N", columns="fct", values="average").plot(
    logy=True, ax=ax[0, 1]
)
cc[cc.fct != "pyfilter_dmax"].pivot(index="N", columns="fct", values="average").plot(
    logy=True, logx=True, ax=ax[1, 1]
)
cc[(cc.fct.str.contains("cfilter") | cc.fct.str.contains("numpy"))].pivot(
    index="N", columns="fct", values="average"
).plot(logy=True, ax=ax[1, 0])
ax[0, 0].set_title("Comparison of filter implementations")
ax[0, 1].set_title("Comparison of filter implementations\nwithout pyfilter_dmax")

Text(0.5, 1.0, 'Comparison of filter implementations\nwithout pyfilter_dmax')

The results depends on the machine, its number of cores, the compilation settings of numpy or this module.