Coverage for /Users/Dave/git_repos/_packages_/python/fundamentals/fundamentals/fmultiprocess.py: 38%

# Coverage for /Users/Dave/git_repos/_packages_/python/fundamentals/fundamentals/fmultiprocess.py : 38%

## 32 statements 12 run 20 missing 0 excluded

Hot-keys on this page

r m x p toggle line displays

j k next/prev highlighted chunk

0 (zero) top of page

1 (one) first highlighted chunk

1#!/usr/local/bin/python

2# encoding: utf-8

3"""

4*A function to quickly add multiprocessing to any program*

5

6:Author:

7 David Young

8

9:Date Created:

10 November 9, 2017

11"""

12 from __future__ import division

13################# GLOBAL IMPORTS ####################

14 from past.utils import old_div

15 import sys

16 import os

17 os.environ['TERM'] = 'vt100'

18 from fundamentals import tools

19 from multiprocess import cpu_count, Pool

20 from functools import partial

21 import inspect

22 import psutil

23

24

25 def fmultiprocess(

26 log,

27 function,

28 inputArray,

29 poolSize=False,

30 timeout=3600,

31 **kwargs):

32 """multiprocess pool

33

34 **Key Arguments:**

35 - ``log`` -- logger

36 - ``function`` -- the function to multiprocess

37 - ``inputArray`` -- the array to be iterated over

38 - ``poolSize`` -- limit the number of CPU that are used in multiprocess job

39 - ``timeout`` -- time in sec after which to raise a timeout error if the processes have not completed

40

41 **Return:**

42 - ``resultArray`` -- the array of results

43

44 **Usage:**

45

46 .. code-block:: python

47

48 from fundamentals import multiprocess

49 # DEFINE AN INPUT ARRAY

50 inputArray = range(10000)

51 results = multiprocess(log=log, function=functionName, poolSize=10, timeout=300,

52 inputArray=inputArray, otherFunctionKeyword="cheese")

53 """

54 log.debug('starting the ``multiprocess`` function')

55

56 # DEFINTE POOL SIZE - NUMBER OF CPU CORES TO USE (BEST = ALL - 1)

57 if not poolSize:

58 poolSize = psutil.cpu_count()

59

60 if poolSize:

61 p = Pool(processes=poolSize)

62 else:

63 p = Pool()

64

65 cpuCount = psutil.cpu_count()

66 chunksize = int(old_div((len(inputArray) + 1), (cpuCount * 3)))

67

68 if chunksize == 0:

69 chunksize = 1

70

71 # chunksize = 1

72

73 # MAP-REDUCE THE WORK OVER MULTIPLE CPU CORES

74 if "log" in inspect.getargspec(function)[0]:

75 mapfunc = partial(function, log=log, **kwargs)

76 resultArray = p.map_async(mapfunc, inputArray, chunksize=chunksize)

77 else:

78 mapfunc = partial(function, **kwargs)

79 resultArray = p.map_async(mapfunc, inputArray, chunksize=chunksize)

80

81 resultArray = resultArray.get(timeout=timeout)

82

83 p.close()

84 p.join()

85 # p.terminate()

86

87 log.debug('completed the ``multiprocess`` function')

88 return resultArray