Geek asks and answers

Multiprocessing and Python Argument

Hey. I am trying to run an example of multiprocessing in documents: http://docs.python.org/3.4/library/concurrent.futures.html , which uses prime numbers, but with a slight difference.

I want to be able to call a function with several arguments. What I do corresponds to small fragments of text (in a list about 30 thousand long) for a much larger fragment of text and returns to where smaller lines begin in a larger line.

I can do it in serial as follows:

matchList = []
for pattern in patterns:

    # Approximate pattern matching
    patternStartingPositions = processPattern(pattern, numMismatchesAllowed, transformedText, charToIndex, countMatrix, firstOccurrence, suffixArray)

    # Now add each starting position found onto our master list.
    for startPos in patternStartingPositions:
        matchList.append(startPos)

But I want to do this to speed things up:

matchList = []
with concurrent.futures.ProcessPoolExecutor() as executor:
    for pattern, res in zip(patterns, executor.map(processPattern(pattern, numMismatchesAllowed, transformedText, charToIndex, countMatrix, firstOccurrence, suffixArray), patterns)):
        print('%d is starts at: %s' % (pattern, res))

At this point, I just got a print call there because I can't get the line above, the process call to work.

, , , 7 , , , .

:

UnboundLocalError: 'pattern', .

.

, , , processPattern:

matchList = []
with concurrent.futures.ProcessPoolExecutor() as executor:
    for pattern, res in zip(patterns, executor.map(processPattern(numMismatchesAllowed, transformedText, charToIndex, countMatrix, firstOccurrence, suffixArray), patterns)):
        print('%d is starts at: %s' % (pattern, res))

:

TypeError: processPattern() 1 : 'suffixArray'.

, pattern !

+3
3

, ( zip) submit, map:

(pattern, executor.submit(processPattern, pattern, ...) for pattern in patterns)

( ), processPatterns, , .submit. ​​ :

with concurrent.futures.ProcessPoolExecutor() as executor:
    for pattern, res in ((pattern, executor.submit(processPattern, pattern, numMismatchesAllowed, transformedText, charToIndex, countMatrix, firstOccurrence, suffixArray)) for pattern in patterns):
        print('%d is starts at: %s' % (pattern, res.result()))
+2

, , executor.map.

with concurrent.futures.ProcessPoolExecutor() as executor:
    # is_prime is the function, PRIMES are the arguments
    for number, prime in zip(PRIMES, executor.map(is_prime, PRIMES)): 
        print('%d is prime: %s' % (number, prime))

processPattern executor.map

with concurrent.futures.ProcessPoolExecutor() as executor:
    for pattern, res in zip(patterns, executor.map(processPattern(numMis... # <- BAD
        print('%d is starts at: %s' % (pattern, res))


with concurrent.futures.ProcessPoolExecutor() as executor:
    for pattern, res in zip(patterns, executor.map(processPattern, <stuff>)):
        print('%d is starts at: %s' % (pattern, res))

<stuff> processPattern .

, , , , , , patterns ( @uhbif19)

EDIT:

<stuff> iterable, , (processPattern ). patterns , , itertools.repeat:

from itertools import repeat
args = (patterns, 
        repeat(numMismatchesAllowed, len(PATTERNS)),
        repeat(transformedText, len(PATTERNS)),
        repeat(charToIndex, len(PATTERNS)),
        <etc...>
        )


for pattern, res in zip(PATTERNS, executor.map(process, *args)):

, , . .

2:

, submit vs map

import concurrent.futures

def process(a, b):
    return a.upper() + b

with concurrent.futures.ProcessPoolExecutor() as executor:
    for c, fut in [(c, executor.submit(process, c, 'b')) for c in 'testing']:
        print(c, fut.result())

with concurrent.futures.ProcessPoolExecutor() as executor:
    for c, res in zip('testing', executor.map(process, 'testing', 'bbbbbbb')):
        print(c, str(res))
+2

Python for-loop , , .

with concurrent.futures.ProcessPoolExecutor() as executor:

    def work(pattern):
        return processPattern(pattern, numMismatchesAllowed, transformedText, charToIndex, countMatrix, firstOccurrence, suffixArray)

    results = executor.map(work, patterns)

    for pattern, res in zip(patterns, results):
        print('%d is starts at: %s' % (pattern, res))

, , continue break, , . :

for i in something:
    work(i)


map(work, something)
+1
source

More articles:


All Articles