Geek asks and answers

How to move this loop to numpy?

I have a slow loop that I want to do (much) faster by clicking it on numpy. I spent days playing with this code, I won’t go anywhere. Is this possible, or is there some kind of trick I don't have? Can I do refactoring to help?

As you can see, I want to summarize mixins as shifted xs.

import numpy as np

blocksize = 1000 # Chosen at runtime.
mixinsize = 100 # Chosen at runtime.
count = 10000 # Chosen at runtime.
xs = np.random.randint(0, blocksize + 1, count) # In practice this is data.
mixins = np.empty((count, mixinsize)) # In practice this is data.

# The slow part:
accumulator = np.zeros(blocksize + mixinsize)
for i in xrange(count):
    accumulator[xs[i]:xs[i] + mixinsize] += mixins[i]

Comments on the accepted answer

  • To get numba to work, I first had to make sure that I am using the appropriate numpy types everywhere and, of course, not the usual python types. This in itself was a huge performance improvement.
  • Numba improved a particular case of speed as the quantity was large, and this is what I wanted (maybe I should have highlighted this in the question).
  • numba , , numba - , , .
  • anaconda numba, , numba.
+3
1

Numba 0.11 ( 0.12) ​​numba.pydata.org. LLVM:

# plain NumPy version
import numpy as np

def foobar(mixinsize, count, xs, mixins, acc):
    for i in xrange(count):
        k = xs[i]
        acc[k:k + mixinsize] += mixins[i,:]


# LLVM compiled version
from numba import jit, void, int64, double
signature = void(int64,int64,int64[:],double[:,:],double[:])
foobar_jit = jit(signature)(foobar)

if __name__ == "__main__":   
    from time import clock

    blocksize = 1000 # Chosen at runtime.
    mixinsize = 100 # Chosen at runtime.
    count = 100000 # Chosen at runtime. 
    xs = np.random.randint(0, blocksize + 1, count)
    mixins = np.empty((count, mixinsize))
    acc = np.zeros(blocksize + mixinsize)

    t0 = clock()
    foobar(mixinsize, count, xs, mixins, acc)
    t1 = clock()
    print("elapsed time: %g ms" % (1000*(t1-t0),))

    t2 = clock()
    foobar_jit(mixinsize, count, xs, mixins, acc)
    t3 = clock()
    print("elapsed time with numba jit: %g ms" % (1000*(t3-t2),))

    print("speedup factor: %g" % ((t1-t0)/(t3-t2),))

$ python test_numba.py
elapsed time: 590.632 ms
elapsed time with numba jit: 12.31 ms
speedup factor: 47.9799

, 50- Python.

C , clang/LLVM .

void foobar(long mixinsize, long count, 
    long *xs, double *mixins, double *accumulator)
{
    long i, j, k;
    double *cur, *acc;   
    for (i=0;i<count;i++) {
        acc = accumulator + xs[i];
        cur = mixins + i*mixinsize;
        for(j=0;j<mixinsize;j++) *acc++ += *cur++;
    }
}

from numpy.ctypeslib import ndpointer
import ctypes
so = ctypes.CDLL('plainc.so')
foobar_c = so.foobar
foobar_c.restype = None
foobar_c.argtypes = (
    ctypes.c_long,
    ctypes.c_long,
    ndpointer(dtype=np.int64, ndim=1),
    ndpointer(dtype=np.float64, ndim=2),
    ndpointer(dtype=np.float64, ndim=1)
)


t4 = clock()
foobar_c(mixinsize, count, xs, mixins, acc)
t5 = clock()
print("elapsed time with plain C: %g ms" % (1000*(t5-t4),))

$ CC -Ofast -shared -m64 -o plainc.so plainc.c
$ python test_numba.py
elapsed time: 599.136 ms
elapsed time with numba jit: 11.958 ms
speedup factor: 50.1034
elapsed time with plain C: 5.472 ms

, Numba , C -Ofast. , -O2 8 . , Numba JIT Python 75% C OO2. Python.

Python:

def foobar_py(mixinsize, count, xs, mixins, acc):
    for i in xrange(count):
        k = xs[i]
        for j in xrange(mixinsize):
            acc[j+k] += mixins[i][j]


# covert NumPy arrays to lists
_xs = map(int,xs)
_mixins = [map(float,mixins[i,:]) for i in xrange(count)]
_acc = map(float,acc)


t6 = clock()
foobar_py(mixinsize, count, _xs, _mixins, _acc)
t7 = clock()
print("elapsed time with plain Python: %g ms" % (1000*(t7-t6),))

Python 1775 . , Python 3- , NumPy, 150- Numba 350- C -Ofast.

, C. A. R. Hoare: " - ". , . , ? ? .

+9

More articles:


All Articles