Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1#!/usr/local/bin/python 

2# encoding: utf-8 

3""" 

4*A function to quickly add multiprocessing to any program* 

5 

6:Author: 

7 David Young 

8 

9:Date Created: 

10 November 9, 2017 

11""" 

12from __future__ import division 

13################# GLOBAL IMPORTS #################### 

14from past.utils import old_div 

15import sys 

16import os 

17os.environ['TERM'] = 'vt100' 

18from fundamentals import tools 

19from multiprocess import cpu_count, Pool 

20from functools import partial 

21import inspect 

22import psutil 

23 

24 

25def fmultiprocess( 

26 log, 

27 function, 

28 inputArray, 

29 poolSize=False, 

30 timeout=3600, 

31 **kwargs): 

32 """multiprocess pool 

33 

34 **Key Arguments:** 

35 - ``log`` -- logger 

36 - ``function`` -- the function to multiprocess 

37 - ``inputArray`` -- the array to be iterated over 

38 - ``poolSize`` -- limit the number of CPU that are used in multiprocess job 

39 - ``timeout`` -- time in sec after which to raise a timeout error if the processes have not completed 

40 

41 **Return:** 

42 - ``resultArray`` -- the array of results 

43 

44 **Usage:** 

45 

46 .. code-block:: python  

47 

48 from fundamentals import multiprocess 

49 # DEFINE AN INPUT ARRAY 

50 inputArray = range(10000) 

51 results = multiprocess(log=log, function=functionName, poolSize=10, timeout=300, 

52 inputArray=inputArray, otherFunctionKeyword="cheese") 

53 """ 

54 log.debug('starting the ``multiprocess`` function') 

55 

56 # DEFINTE POOL SIZE - NUMBER OF CPU CORES TO USE (BEST = ALL - 1) 

57 if not poolSize: 

58 poolSize = psutil.cpu_count() 

59 

60 if poolSize: 

61 p = Pool(processes=poolSize) 

62 else: 

63 p = Pool() 

64 

65 cpuCount = psutil.cpu_count() 

66 chunksize = int(old_div((len(inputArray) + 1), (cpuCount * 3))) 

67 

68 if chunksize == 0: 

69 chunksize = 1 

70 

71 # chunksize = 1 

72 

73 # MAP-REDUCE THE WORK OVER MULTIPLE CPU CORES 

74 if "log" in inspect.getargspec(function)[0]: 

75 mapfunc = partial(function, log=log, **kwargs) 

76 resultArray = p.map_async(mapfunc, inputArray, chunksize=chunksize) 

77 else: 

78 mapfunc = partial(function, **kwargs) 

79 resultArray = p.map_async(mapfunc, inputArray, chunksize=chunksize) 

80 

81 resultArray = resultArray.get(timeout=timeout) 

82 

83 p.close() 

84 p.join() 

85 # p.terminate() 

86 

87 log.debug('completed the ``multiprocess`` function') 

88 return resultArray