PyXMake Developer Guide: D:/03_Workspaces/01_Eclipse/pyx_core/PyXMake/Tools/Utility.py Source File

| PyXMake Developer Guide 1.0 PyXMake

Utility.py
1 # -*- coding: utf-8 -*-
2 # %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
3 # % Utility - Classes and Functions %
4 # %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
5 """
6 Classes and functions defined for convenience.
7
8 @note: PyCODAC module
9 Created on 15.07.2016
10
11 @version: 1.0
12 ----------------------------------------------------------------------------------------------
13 @requires:
14 -
15
16 @change:
17 -
18
19 @author: garb_ma [DLR-FA,STM Braunschweig]
20 ----------------------------------------------------------------------------------------------
21 """
22
23 ## @package PyXMake.Tools.Utility
24 # Module of basic functions.
25 ## @author
26 # Marc Garbade
27 ## @date
28 # 15.07.2017
29 ## @par Notes/Changes
30 # - Added documentation // mg 29.03.2018
31
32 from future import standard_library
33 try:
34 standard_library.install_aliases()
35 except:
36 pass
37
38 try:
39 from builtins import str
40 from builtins import object
41 except ImportError:
42 pass
43
44 from past.builtins import long
45 from past.builtins import unicode
46
47 import sys
48 import os
49 import io
50 import six
51 import shutil
52 import subprocess
53 import platform
54 import struct
55 import posixpath, ntpath
56
57 import numpy as np
58 import random as rd
59 import tempfile
60 import zipfile
61
62 try:
63 import cPickle as cp # @UnusedImport
64 except:
65 import pickle as cp # @Reimport
66
67 try:
68 from contextlib import contextmanager # @UnusedImport
69 except ImportError:
70 from contextlib2 import contextmanager # @Reimport
71
72 from types import MethodType
73 from .. import PyXMakePath
74
75 ## @class PyXMake.Tools.Utility.ChangedWorkingDirectory
76 # Class to create 2to3 compatible pickling dictionary. Inherited from built-in object.
77 class ChangedWorkingDirectory(object):
78 """
79 Context manager for temporarily changing the current working directory.
80
81 @author: Brian M. Hunt
82 """
83 def __init__(self, newPath):
84 self.newPath = os.path.expanduser(newPath)
85 # Create target directory & all intermediate directories if don't exists
86 if not os.path.exists(self.newPath) and self.newPath != os.getcwd():
87 print("==================================")
88 print("Creating a new scratch folder @:", self.newPath)
89 print("This folder will not be deleted once the job is done!")
90 print("==================================")
91 os.makedirs(self.newPath)
92
93 def __enter__(self):
94 self.savedPath = os.getcwd()
95 os.chdir(self.newPath)
96
97 def __exit__(self, etype, value, traceback):
98 os.chdir(self.savedPath)
99
100 ## @class PyXMake.Tools.Utility.GetDataFromPickle
101 # Class to create 2to3 compatible pickling dictionary. Inherited from built-in object.
102 class GetDataFromPickle(object):
103 """
104 Class to convert an arbitrary pickle file (2.x & 3.x) into a readable
105 dictionary.
106 """
107 def __init__(self, FileName):
108 """
109 Get a dictionary from a *.cpd file.
110
111 @param: self, FileName
112 @type: self: object
113 @type: FileName: string
114 """
115 ## Dictionary for further processing.
116 self.Data = GetDataFromPickle.getDictfromFile(FileName)
117 os.remove(FileName)
118
119 @staticmethod
120 def getDictfromFile(FileName):
121 """
122 Open a *.cpd file and extract the dictionary stored within.
123
124 @param: FileName
125 @type: FileName: string
126 """
127 FileIn = open(FileName, "rb")
128 Dict = cp.load(FileIn)
129 FileIn.close()
130 return Dict
131
132 ## @class PyXMake.Tools.Utility.UpdateZIP
133 # Class to create 2to3 compatible pickling dictionary. Inherited from built-in object.
134 class UpdateZIP(object):
135 """
136 Context manager for update an existing ZIP folder
137
138 @author: Marc Garbade
139 """
140 def __init__(self, zipname, zipdata, outpath=os.getcwd(), exclude=[], update=True):
141 self.ZipName = zipname
142 self.ZipData = zipdata
143
144 # Collect content of the ZIP folder from input
145 self.buffer = open(os.path.join(os.getcwd(), self.ZipName), 'wb+')
146 shutil.copyfileobj(self.ZipData, self.buffer)
147 self.buffer.close();
148
149 # Initialize local variables
150 self.Output = io.BytesIO();
151 self.OutputPath = outpath
152 self.ExcludeFiles = exclude
153
154 self.Update = update
155
156 def __enter__(self):
157 # Extract data in current workspace and to examine its content
158 with zipfile.ZipFile(str(self.ZipName)) as Input:
159 Input.extractall()
160 os.remove(self.ZipName)
161
162 # Do not copy input files back into the new zip folder.
163 if not self.Update:
164 self.ExcludeFiles.extend([f for f in os.listdir(os.getcwd()) if os.path.isfile(os.path.join(os.getcwd(), f))])
165
166 def __exit__(self, etype, value, traceback):
167 # Collect all newly created files and store them in a memory ZIP folder.
168 with zipfile.ZipFile(self.Output,"w", zipfile.ZIP_DEFLATED) as Patch:
169 for dirpath, _, filenames in os.walk(os.getcwd()):
170 for f in filenames:
171 filepath = os.path.join(dirpath,f)
172 # Add all result files to the zip folder. Ignore old zip and object files
173 if not f.endswith((".zip", ".obj")) and f not in self.ExcludeFiles:
174 Patch.write(filepath, f)
175
176 # Write content of memory ZIP folder to disk (for download). Everything else has been removed by now.
177 with open(os.path.join(self.OutputPath,self.ZipName), "wb") as f:
178 f.write(self.Output.getvalue())
179
180 @contextmanager
181 def TemporaryDirectory(default=None):
182 """
183 Create a temporary dictionary for use with the "with" statement. Its content is deleted after execution.
184
185 @param: default
186 @type: default: string
187 """
188 @contextmanager
189 def Changed(newdir, cleanup=lambda: True):
190 """
191 Local helper function to clean up the directory
192 """
193 prevdir = os.getcwd()
194 os.chdir(os.path.expanduser(newdir))
195 try:
196 yield
197 finally:
198 os.chdir(prevdir)
199 cleanup()
200 # Create a new temporary folder in default.
201 # Uses platform-dependent defaults when set to None.
202 dirpath = tempfile.mkdtemp(dir=default)
203 def cleanup():
204 shutil.rmtree(dirpath)
205 with Changed(dirpath, cleanup):
206 yield dirpath
207
208 @contextmanager
209 def ConsoleRedirect(to=os.devnull, stdout=None):
210 """
211 Redirect console output to a given file.
212 """
213 def fileno(file_or_fd):
214 """
215 Small helper function to check the validity of the dump object.
216 """
217 fd = getattr(file_or_fd, 'fileno', lambda: file_or_fd)()
218 if not isinstance(fd, int):
219 raise ValueError("Expected a file (`.fileno()`) or a file descriptor")
220 return fd
221
222 def flush(stream):
223 """
224 Also flush c stdio buffers on python 3 (if possible)
225 """
226 try:
227 import ctypes
228 from ctypes.util import find_library
229 except ImportError:
230 libc = None
231 else:
232 try:
233 libc = ctypes.cdll.msvcrt # Windows
234 except OSError:
235 libc = ctypes.cdll.LoadLibrary(find_library('c'))
236 try:
237 # Flush output associated with C/C++
238 libc.fflush(ctypes.c_void_p.in_dll(libc, 'stdout'))
239 except (AttributeError, ValueError, IOError):
240 pass # unsupported
241
242 # Regular flush
243 stream.flush()
244 pass
245
246 if stdout is None:
247 stdout = sys.stdout
248
249 stdout_fd = fileno(stdout)
250 # copy stdout_fd before it is overwritten
251 #NOTE: `copied` is inheritable on Windows when duplicating a standard stream
252 with os.fdopen(os.dup(stdout_fd), stdout.mode) as copied:
253 flush(stdout) # flush library buffers that dup2 knows nothing about
254 try:
255 os.dup2(fileno(to), stdout_fd) # $ exec >&to
256 except ValueError: # filename
257 with open(to, 'wb') as to_file:
258 os.dup2(to_file.fileno(), stdout_fd) # $ exec > to
259 try:
260 yield stdout # allow code to be run with the redirected stdout
261 finally:
262 # restore stdout to its previous value
263 #NOTE: dup2 makes stdout_fd inheritable unconditionally
264 flush(stdout)
265 os.dup2(copied.fileno(), stdout_fd) # $ exec >&copied
266
267 @contextmanager
268 def MergedConsoleRedirect(f): # $ exec 2>&1
269 """
270 Redirect all console outputs to a given stream
271 """
272 with ConsoleRedirect(to=sys.stdout, stdout=sys.stdin) as inp, ConsoleRedirect(to=sys.stdout, stdout=sys.stderr) as err, ConsoleRedirect(to=f,stdout=sys.stdout) as out:
273 yield (inp, err, out)
274
275 @contextmanager
276 def FileOutput(FileName): # $ exec 2>&1
277 """
278 Redirect outputs to a given file.
279 """
280 if sys.version_info >= (3,4):
281 import contextlib
282 # Python 3.4 and higher
283 with open(FileName, 'w', encoding="utf-8") as f, contextlib.redirect_stdout(f), MergedConsoleRedirect(sys.stdout):
284 yield f
285 else:
286 # Lower version (unstable and deprecated)
287 with open(FileName, 'w', encoding="utf-8") as f, MergedConsoleRedirect(f):
288 yield f
289
290 def GetPyXMakePath():
291 """
292 Get the PyXMake path from *__init__.
293 """
294 Path = PyXMakePath
295 return Path
296
297 def GetPlatform():
298 """
299 Get the underlying machine platform in lower cases.
300 """
301 return str(platform.system()).lower()
302
303 def GetArchitecture():
304 """
305 Get the underlying machine architecture. Returns either x86 or x64 which corresponds to
306 32 or 64 bit systems.
307 """
308 if struct.calcsize("P") * 8 == 64:
309 arch = 'x64'
310 else:
311 arch = 'x86'
312 return arch
313
314 def GetTemporaryFileName(arg=None, filename="Temp", extension=".cpd", **kwargs):
315 """
316 Create a temporary file name with extension *.cpd by default. Optional argument: Seed for random number generation.
317 """
318 if isinstance(arg, (int, long)):
319 _seed = arg
320 rd.seed(_seed)
321 randTempInteger = rd.randint(1,1000)
322 # Added backwards compatibility
323 TempFileName = filename + str(randTempInteger) + kwargs.get("ending",extension)
324 else:
325 rd.seed()
326 randTempInteger = rd.randint(1,1000)
327 # Added backwards compatibility
328 TempFileName = filename + str(randTempInteger) + kwargs.get("ending",extension)
329 return TempFileName
330
331 def GetIterableAsList(Iterable):
332 """
333 Walk through an iterable input set and store the results in a list.
334 """
335 AsList = []
336 for i in range(0,len(Iterable)):
337 AsList.append(Iterable[i])
338 return AsList
339
340 def IsNotEmpty(s):
341 """
342 Check whether a string is empty and/or not given. Returns True otherwise.
343 """
344 return bool(s and s.strip())
345
346 def AsDrive(s, sep=os.path.sep):
347 """
348 Return s as drive to start an absolute path with path.join(...).
349 """
350 if sep != ntpath.sep:
351 # Linux
352 drive = posixpath.join(posixpath.sep,s)
353 else:
354 # Windows
355 drive = os.path.join(s+":",os.path.sep)
356 return drive
357
358 def Popen(command, verbosity):
359 """
360 Run command line string "command" in a separate subprocess.
361 Show output in current console window in dependence of verbosity level:
362 - 0 --> Quiet
363 - 1 --> Only show errors
364 - 2 --> Show every command line output.
365
366 @author: garb_ma
367 @param: command, verbosity
368 @type: string, integer
369 """
370 p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
371 stdout, stderr = p.communicate()
372 # Output standard and error messages in dependence of verbosity level.
373 if verbosity >= 2 and IsNotEmpty(stdout):
374 print((stdout.decode('utf-8',errors='ignore').replace('\n', '')))
375 if verbosity >= 1 and IsNotEmpty(stderr):
376 print((stderr.decode('utf-8',errors='ignore').replace('\n', '')))
377 # Return subprocess
378 return p
379
380 def SSHPopen(ssh_client, command, verbosity):
381 """
382 Run command line string "command" in a separate SSH client process.
383 Show output in current console window in dependence of verbosity level:
384 - 0 --> Quiet
385 - 1 --> Only show errors
386 - 2 --> Show every command line output.
387
388 @author: garb_ma
389 @param: command, verbosity
390 @type: string, integer
391 """
392 _, stdout, stderr = ssh_client.exec_command(command)
393 sexit = stdout.channel.recv_exit_status()
394
395 sout = stdout.readlines()
396 serr = stderr.readlines()
397
398 if verbosity >= 2 and sout:
399 print("".join(sout))
400 if verbosity >= 1 and serr:
401 print("".join(serr))
402
403 return sexit
404
405 def ConcatenateFiles(filename, files, source=os.getcwd(), ending=''):
406 """
407 Concatenate all files into one.
408 """
409 FileRead = 0
410
411 # Concatenate all files into one temporary file
412 with open(filename,'wb') as wfd:
413 for f in [os.path.join(source,cs) if IsNotEmpty(str(os.path.splitext(cs)[1]))
414 else os.path.join(source,cs+ending) for cs in files]:
415 if FileRead > 0:
416 # Add a empty line between two source file includes
417 wfd.write("\n\n".encode())
418 with open(f,'rb') as fd:
419 shutil.copyfileobj(fd, wfd, 1024*1024*10)
420 FileRead += 1
421
422 def ReplaceTextinFile(filename, outname, replace, inend='', outend='', source=os.getcwd()):
423 """
424 Replace all occurrences of replace in filename.
425 """
426 Inputfile = os.path.join(source,filename+inend)
427 Outputfile = outname+outend
428 with open(Inputfile) as infile, open(Outputfile, 'w') as outfile:
429 for line in infile:
430 for src, target in replace.items():
431 line = line.replace(src, target)
432 outfile.write(line)
433
434 def DeleteFilesbyEnding(identifier):
435 """
436 Delete all files from workspace
437
438 @author: Marc Garbade, 26.02.2018
439
440 @param: identifier: A tuple specifying the files to remove.
441 @type: Tuple
442 """
443 for f in os.listdir(os.getcwd()):
444 if f.endswith(identifier):
445 os.remove(f)
446
447 def AddFunctionToObject(_func, _obj):
448 """
449 Bind a function to an existing object.
450 """
451 return MethodType(_func, _obj)
452
453 def PrepareObjectforPickling(_obj):
454 """
455 Prepare a object for pickling and convert all numpy
456 arrays to python defaults (2to3 compatible).
457 """
458 _dictobj = _obj.__dict__.copy()
459 _dictobj['__np_obj_path'] = []
460 for path, value in ObjectWalk(_dictobj):
461 if isinstance(value, (np.ndarray, np.generic)):
462 parent = _dictobj
463 for step in path[:-1]:
464 parent = parent[step]
465 parent[path[-1]] = value.tolist()
466 _dictobj['__np_obj_path'].append(path[-1])
467 return _dictobj
468
469 def RecoverDictionaryfromPickling(_dict):
470 """
471 Restore the original dictionary by converting python defaults to their
472 numpy equivalents if required (2to3 compatible).
473 """
474 _dictobj = _dict.copy()
475 # Recreate serialized arrays accordingly
476 for key, value in _dictobj.items():
477 if key in _dictobj['__np_obj_path']:
478 _dictobj[key] = np.float64(value)
479 if len(np.shape(value)) == 2:
480 _dictobj[key] = np.asmatrix(value)
481 else:
482 pass
483 else:
484 pass
485 return _dictobj
486
487 def PathLeaf(path):
488 """
489 Return the last item of an arbitrary path (its leaf).
490 """
491 head, tail = ntpath.split(path)
492 return tail or ntpath.basename(head)
493
494 def ArbitraryFlattening(container):
495 """
496 Restore the original dictionary by converting python defaults to their
497 numpy equivalents if required (2to3 compatible).
498 """
499 for i in container:
500 if isinstance(i, (list,tuple, np.ndarray)):
501 for j in ArbitraryFlattening(i):
502 yield j
503 else:
504 yield i
505
506 def FileWalk(source, path=os.getcwd()):
507 """
508 Walk recursively through path. Check if all files listed in source are present.
509 If True, return them. If False, return all files present in the given path.
510 """
511 # Initialize all variables
512 files = list([])
513 # These are all uploaded files
514 InputFiles = [f for f in os.listdir(path) if os.path.isfile(os.path.join(path, f))]
515 try:
516 # Check if source code files have been specified. Otherwise, use everything from the ZIP file
517 if all(np.array([len(source),len(InputFiles)]) >= 2):
518 # Ignore file extension here. This will interfere with the checks later on.
519 if set([os.path.splitext(x)[0] for x in source]).issubset(set([os.path.splitext(f)[0] for f in InputFiles])):
520 files.extend(source)
521 else:
522 pass
523 else:
524 files.extend(InputFiles)
525 except TypeError:
526 files.extend(InputFiles)
527 pass
528
529 return files
530
531 def PathWalk(path, exclude=False, startswith=None, endswith=None, contains=None):
532 """
533 Walk recursively through path. Exclude both folders and files if requested.
534 """
535 def Skip(x, starts, contains, ends):
536 """
537 Evaluate skip condition
538 """
539 if isinstance(starts, (six.string_types, tuple)):
540 if x.startswith(starts):
541 return True
542 if isinstance(contains, (six.string_types, tuple, list)):
543 tmp = list([]); tmp.append(contains)
544 tmp = tuple(ArbitraryFlattening(tmp))
545 if any(s in x for s in tmp):
546 return True
547 if isinstance(ends, (six.string_types, tuple)):
548 if x.endswith(ends):
549 return True
550 return False
551
552 for root, dirs, files in os.walk(os.path.normpath(path)):
553 if exclude:
554 files = [f for f in files if not Skip(f,startswith,contains,endswith)]
555 dirs[:] = [d for d in dirs if not Skip(d,startswith,contains,endswith)]
556 yield root, dirs, files
557
558 def ObjectWalk(obj, path=(), memo=None):
559 """
560 Walk recursively through nested python objects.
561
562 @author: Yaniv Aknin, 13.12.2011
563
564 @param: obj, path, memo
565 @type: object, list, boolean
566 """
567 string_types = (str, unicode)
568 iteritems = lambda mapping: getattr(mapping, 'iteritems', mapping.items)()
569 if memo is None:
570 memo = set()
571 iterator = None
572 if isinstance(obj, dict):
573 iterator = iteritems
574 elif isinstance(obj, (list, set)) and not isinstance(obj, string_types):
575 iterator = enumerate
576 if iterator:
577 if id(obj) not in memo:
578 memo.add(id(obj))
579 for path_component, value in iterator(obj):
580 for result in ObjectWalk(value, path + (path_component,), memo):
581 yield result
582 memo.remove(id(obj))
583 else:
584 yield path, obj
585
586 if __name__ == '__main__':
587 pass
PyXMake.Tools.Utility.GetArchitecture
def GetArchitecture()
Definition: Utility.py:303
PyXMake.Tools.Utility.RecoverDictionaryfromPickling
def RecoverDictionaryfromPickling(_dict)
Definition: Utility.py:469
PyXMake.Tools.Utility.ConsoleRedirect
def ConsoleRedirect(to=os.devnull, stdout=None)
Definition: Utility.py:209
PyXMake.Tools.Utility.UpdateZIP.OutputPath
OutputPath
Definition: Utility.py:151
PyXMake.Tools.Utility.AsDrive
def AsDrive(s, sep=os.path.sep)
Definition: Utility.py:346
PyXMake.Tools.Utility.SSHPopen
def SSHPopen(ssh_client, command, verbosity)
Definition: Utility.py:380
PyXMake.Tools.Utility.FileOutput
def FileOutput(FileName)
Definition: Utility.py:276
PyXMake.Tools.Utility.UpdateZIP.ZipData
ZipData
Definition: Utility.py:142
PyXMake.Tools.Utility.UpdateZIP.ZipName
ZipName
Definition: Utility.py:141
PyXMake.Tools.Utility.GetDataFromPickle.Data
Data
Dictionary for further processing.
Definition: Utility.py:116
PyXMake.Tools.Utility.AddFunctionToObject
def AddFunctionToObject(_func, _obj)
Definition: Utility.py:447
PyXMake.Tools.Utility.UpdateZIP
Class to create 2to3 compatible pickling dictionary.
Definition: Utility.py:134
PyXMake.Tools.Utility.UpdateZIP.Output
Output
Definition: Utility.py:150
PyXMake.Tools.Utility.UpdateZIP.ExcludeFiles
ExcludeFiles
Definition: Utility.py:152
PyXMake.Tools.Utility.TemporaryDirectory
def TemporaryDirectory(default=None)
Definition: Utility.py:181
PyXMake.Tools.Utility.GetIterableAsList
def GetIterableAsList(Iterable)
Definition: Utility.py:331
PyXMake.Tools.Utility.DeleteFilesbyEnding
def DeleteFilesbyEnding(identifier)
Definition: Utility.py:434
PyXMake.Tools.Utility.ChangedWorkingDirectory
Class to create 2to3 compatible pickling dictionary.
Definition: Utility.py:77
PyXMake.Tools.Utility.PathLeaf
def PathLeaf(path)
Definition: Utility.py:487
PyXMake.Tools.Utility.GetDataFromPickle
Class to create 2to3 compatible pickling dictionary.
Definition: Utility.py:102
PyXMake.Tools.Utility.ChangedWorkingDirectory.newPath
newPath
Definition: Utility.py:84
PyXMake.Tools.Utility.GetTemporaryFileName
def GetTemporaryFileName(arg=None, filename="Temp", extension=".cpd", kwargs)
Definition: Utility.py:314
PyXMake.Tools.Utility.GetDataFromPickle.getDictfromFile
def getDictfromFile(FileName)
Definition: Utility.py:120
PyXMake.Tools.Utility.ObjectWalk
def ObjectWalk(obj, path=(), memo=None)
Definition: Utility.py:558
PyXMake.Tools.Utility.ConcatenateFiles
def ConcatenateFiles(filename, files, source=os.getcwd(), ending='')
Definition: Utility.py:405
PyXMake.Tools.Utility.IsNotEmpty
def IsNotEmpty(s)
Definition: Utility.py:340
PyXMake.Tools.Utility.ReplaceTextinFile
def ReplaceTextinFile(filename, outname, replace, inend='', outend='', source=os.getcwd())
Definition: Utility.py:422
PyXMake.Tools.Utility.GetDataFromPickle.__init__
def __init__(self, FileName)
Definition: Utility.py:107
PyXMake.Tools.Utility.ArbitraryFlattening
def ArbitraryFlattening(container)
Definition: Utility.py:494
PyXMake.Tools.Utility.GetPyXMakePath
def GetPyXMakePath()
Definition: Utility.py:290
PyXMake.Tools.Utility.UpdateZIP.buffer
buffer
Definition: Utility.py:145
PyXMake.Tools.Utility.ChangedWorkingDirectory.savedPath
savedPath
Definition: Utility.py:94
PyXMake.Tools.Utility.MergedConsoleRedirect
def MergedConsoleRedirect(f)
Definition: Utility.py:268
PyXMake.Tools.Utility.PrepareObjectforPickling
def PrepareObjectforPickling(_obj)
Definition: Utility.py:453
PyXMake.Tools.Utility.FileWalk
def FileWalk(source, path=os.getcwd())
Definition: Utility.py:506
PyXMake.Tools.Utility.Popen
def Popen(command, verbosity)
Definition: Utility.py:358
PyXMake.Tools.Utility.PathWalk
def PathWalk(path, exclude=False, startswith=None, endswith=None, contains=None)
Definition: Utility.py:531
PyXMake.Tools.Utility.GetPlatform
def GetPlatform()
Definition: Utility.py:297
PyXMake.Tools.Utility.UpdateZIP.Update
Update
Definition: Utility.py:154