src.jupyter_process_manager.class_processes_manager API documentation
Description: Main module with class which helps in handing many processes

# Module `src.jupyter_process_manager.class_processes_manager`

Main module with class which helps in handing many processes

Expand source code

    """Main module with class which helps in handing many processes"""
    # Standard library imports
    import os
    import sys
    import logging
    from collections import OrderedDict
    from time import sleep
    import datetime

    # Third party imports
    from char import char
    from IPython.display import clear_output
    from IPython.display import display
    from IPython import get_ipython
    from IPython.display import HTML
    from tqdm.auto import tqdm
    from tabulate import tabulate
    from yaspin import yaspin

    # Local imports
    from .class_one_process import OneProcess
    from .other import timedelta_nice_format
    from . import widget

    class JupyterProcessesManager(object):
        """Class to handle creation and showing output for many processes"""

        @char
        def __init__(
                self,
                str_dir_for_output,
                is_to_delete_previous_outputs=True
        ):
            """Initialize object

            Args:
                str_dir_for_output (str): Path to dir where to store output files
                is_to_delete_previous_outputs (bool, optional): \
                    Flag if to delete previous output files
            """
            self.int_processes = 0
            # Create directory where to store processes output
            str_processes_output_dir = os.path.join(
                str_dir_for_output, "processes_output")
            if not os.path.exists(str_processes_output_dir):
                os.makedirs(str_processes_output_dir)
            self.str_dir_for_output = str_processes_output_dir
            if is_to_delete_previous_outputs:
                self._delete_all_previous_outputs()
            self.dict_all_processes_by_id = OrderedDict()
            self.dict_alive_processes_by_id = OrderedDict()
            self.dt_processes_started_at = None

        @char
        def add_function_to_processing(self, func_to_process, *args, **kwargs):
            """Start running function as process

            Args:
                func_to_process (function): Function to add for processing
            """
            new_process = OneProcess(self.str_dir_for_output)
            new_process.start_process(func_to_process, *args, **kwargs)
            if not self.dict_alive_processes_by_id:
                self.dt_processes_started_at = datetime.datetime.now()
            self.dict_all_processes_by_id[new_process.int_process_id] = new_process
            self.dict_alive_processes_by_id[new_process.int_process_id] = new_process

        @char
        def debug_run_of_1_function(self, func_to_process, *args, **kwargs):
            """"""
            new_process = OneProcess(self.str_dir_for_output)
            new_process.debug_run_of_the_func(func_to_process, *args, **kwargs)
            self.dict_all_processes_by_id[new_process.int_process_id] = new_process
            self.dict_alive_processes_by_id[new_process.int_process_id] = new_process

        @char
        def wait_till_all_processes_are_over(
                self,
                int_seconds_step=10,
                int_max_processes_to_show=20
        ):
            """"""
            clear_output(wait=True)
            self.print_info_about_running_processes(
                int_max_processes_to_show=int_max_processes_to_show)
            try:
                while True:
                    if not self.dict_alive_processes_by_id:
                        break
                    with yaspin() as sp:
                        for i in range(int_seconds_step):
                            sp.text = "Updating in {} seconds".format(
                                int_seconds_step - i)
                            sleep(1)
                    clear_output(wait=True)
                    self.print_info_about_running_processes(
                        int_max_processes_to_show=int_max_processes_to_show)
            except KeyboardInterrupt:
                clear_output(wait=True)
                print("Interrupting running processes")
                self.terminate_all_alive_processes()
                print("---> Done")
            print("All processes were finished")

        @char
        def terminate_all_alive_processes(self):
            """"""
            print("Terminating all alive processes")
            for process_num in list(self.dict_alive_processes_by_id):
                print("---> Terminate process: ", process_num + 1)
                process_obj = self.dict_alive_processes_by_id[process_num]
                process_obj.terminate()
                if not process_obj.is_alive():
                    self.dict_alive_processes_by_id.pop(process_num, None)
                    print("------> Done")

        @char
        def show_jupyter_widget(
                self,
                int_seconds_step=10,
                int_max_processes_to_show=20
        ):
            """"""
            clear_output(wait=True)
            APP_GUI = widget.create_jupyter_widget(self)
            display(APP_GUI)
            with widget.OUTPUT_PROCESSES_CONDITIONS:
                self.print_info_about_running_processes(
                    int_max_processes_to_show=int_max_processes_to_show)
            try:
                while True:
                    if not self.dict_alive_processes_by_id:
                        break
                    for i in range(int_seconds_step):
                        sleep(1)
                        get_ipython().kernel.do_one_iteration()
                    widget.OUTPUT_PROCESSES_CONDITIONS.clear_output(wait=True)
                    with widget.OUTPUT_PROCESSES_CONDITIONS:
                        self.print_info_about_running_processes(
                            int_max_processes_to_show=int_max_processes_to_show)
            except KeyboardInterrupt:
                clear_output(wait=True)
                print("Interrupting running processes")
                self.terminate_all_alive_processes()
                print("---> Done")
            print("All processes were finished")

        @char
        def print_info_about_running_processes(self, int_max_processes_to_show=20):
            """"""
            display(HTML("<h2>Processes conditions:</h2>"))

            # print("Conditions of the processes:")
            if self.dt_processes_started_at:
                timedelta = datetime.datetime.now() - self.dt_processes_started_at
                print("Working for:", timedelta_nice_format(timedelta))
            self.print_table_with_conditions(int_max_processes_to_show=20)
            print(
                "ALIVE PROCESSES: ",
                len(self.dict_alive_processes_by_id), "/",
                len(self.dict_all_processes_by_id))

        @char
        def print_table_with_conditions(self, int_max_processes_to_show=20):
            """"""
            list_list_processes_info = []
            list_headers = ["Process Id", "Status", "Runtime"]
            for process_num in list(self.dict_all_processes_by_id):
                list_process_info = []
                process_obj = self.dict_all_processes_by_id[process_num]
                # Delete if process is not alive
                if process_num in list(self.dict_alive_processes_by_id):
                    if not process_obj.is_alive():
                        self.dict_alive_processes_by_id.pop(process_num, None)
                # "Process Id"
                list_process_info.append(process_num)
                # "Status"
                list_process_info.append(process_obj.str_status)
                # "Runtime"
                str_runtime = process_obj.get_how_long_this_process_is_running()
                list_process_info.append(str_runtime)
                list_list_processes_info.append(list_process_info)
            if len(list_list_processes_info) > int_max_processes_to_show:
                list_list_additional_columns = []
                list_list_additional_columns.append(["---", "---", "---"])
                str_num = str(
                    len(list_list_processes_info) - int_max_processes_to_show)
                list_list_additional_columns.append(
                    ["Processes", "Hidden", str_num])
                list_list_additional_columns.append(["---", "---", "---"])
                list_list_processes_info = \
                    list_list_additional_columns + list_list_processes_info[:20]
            # github  psql  orgtbl  pretty
            print(tabulate(
                list_list_processes_info, headers=list_headers, tablefmt="pretty"))

        def _delete_all_previous_outputs(self):
            """Delete outputs of previous processes"""
            for str_filename in os.listdir(self.str_dir_for_output):
                str_file_path = os.path.join(self.str_dir_for_output, str_filename)
                if not os.path.isfile(str_file_path):
                    continue
                if "stdout_" in str_filename or "stderr_" in str_filename:
                    try:
                        os.remove(str_file_path)
                    except Exception as ex:
                        print("Cant DELETE previous thread file: ", str_filename)
                        print(ex)

## Classes

`class JupyterProcessesManager (str_dir_for_output, is_to_delete_previous_outputs=True)`

  Class to handle creation and showing output for many processes

  Initialize object

  ## Args

  `str_dir_for_output` :`str`
    Path to dir where to store output files
  `is_to_delete_previous_outputs` :`bool`, optional

                Flag if to delete previous output files

  Expand source code

      class JupyterProcessesManager(object):
          """Class to handle creation and showing output for many processes"""

          @char
          def __init__(
                  self,
                  str_dir_for_output,
                  is_to_delete_previous_outputs=True
          ):
              """Initialize object

              Args:
                  str_dir_for_output (str): Path to dir where to store output files
                  is_to_delete_previous_outputs (bool, optional): \
                      Flag if to delete previous output files
              """
              self.int_processes = 0
              # Create directory where to store processes output
              str_processes_output_dir = os.path.join(
                  str_dir_for_output, "processes_output")
              if not os.path.exists(str_processes_output_dir):
                  os.makedirs(str_processes_output_dir)
              self.str_dir_for_output = str_processes_output_dir
              if is_to_delete_previous_outputs:
                  self._delete_all_previous_outputs()
              self.dict_all_processes_by_id = OrderedDict()
              self.dict_alive_processes_by_id = OrderedDict()
              self.dt_processes_started_at = None

          @char
          def add_function_to_processing(self, func_to_process, *args, **kwargs):
              """Start running function as process

              Args:
                  func_to_process (function): Function to add for processing
              """
              new_process = OneProcess(self.str_dir_for_output)
              new_process.start_process(func_to_process, *args, **kwargs)
              if not self.dict_alive_processes_by_id:
                  self.dt_processes_started_at = datetime.datetime.now()
              self.dict_all_processes_by_id[new_process.int_process_id] = new_process
              self.dict_alive_processes_by_id[new_process.int_process_id] = new_process

          @char
          def debug_run_of_1_function(self, func_to_process, *args, **kwargs):
              """"""
              new_process = OneProcess(self.str_dir_for_output)
              new_process.debug_run_of_the_func(func_to_process, *args, **kwargs)
              self.dict_all_processes_by_id[new_process.int_process_id] = new_process
              self.dict_alive_processes_by_id[new_process.int_process_id] = new_process

          @char
          def wait_till_all_processes_are_over(
                  self,
                  int_seconds_step=10,
                  int_max_processes_to_show=20
          ):
              """"""
              clear_output(wait=True)
              self.print_info_about_running_processes(
                  int_max_processes_to_show=int_max_processes_to_show)
              try:
                  while True:
                      if not self.dict_alive_processes_by_id:
                          break
                      with yaspin() as sp:
                          for i in range(int_seconds_step):
                              sp.text = "Updating in {} seconds".format(
                                  int_seconds_step - i)
                              sleep(1)
                      clear_output(wait=True)
                      self.print_info_about_running_processes(
                          int_max_processes_to_show=int_max_processes_to_show)
              except KeyboardInterrupt:
                  clear_output(wait=True)
                  print("Interrupting running processes")
                  self.terminate_all_alive_processes()
                  print("---> Done")
              print("All processes were finished")

          @char
          def terminate_all_alive_processes(self):
              """"""
              print("Terminating all alive processes")
              for process_num in list(self.dict_alive_processes_by_id):
                  print("---> Terminate process: ", process_num + 1)
                  process_obj = self.dict_alive_processes_by_id[process_num]
                  process_obj.terminate()
                  if not process_obj.is_alive():
                      self.dict_alive_processes_by_id.pop(process_num, None)
                      print("------> Done")

          @char
          def show_jupyter_widget(
                  self,
                  int_seconds_step=10,
                  int_max_processes_to_show=20
          ):
              """"""
              clear_output(wait=True)
              APP_GUI = widget.create_jupyter_widget(self)
              display(APP_GUI)
              with widget.OUTPUT_PROCESSES_CONDITIONS:
                  self.print_info_about_running_processes(
                      int_max_processes_to_show=int_max_processes_to_show)
              try:
                  while True:
                      if not self.dict_alive_processes_by_id:
                          break
                      for i in range(int_seconds_step):
                          sleep(1)
                          get_ipython().kernel.do_one_iteration()
                      widget.OUTPUT_PROCESSES_CONDITIONS.clear_output(wait=True)
                      with widget.OUTPUT_PROCESSES_CONDITIONS:
                          self.print_info_about_running_processes(
                              int_max_processes_to_show=int_max_processes_to_show)
              except KeyboardInterrupt:
                  clear_output(wait=True)
                  print("Interrupting running processes")
                  self.terminate_all_alive_processes()
                  print("---> Done")
              print("All processes were finished")

          @char
          def print_info_about_running_processes(self, int_max_processes_to_show=20):
              """"""
              display(HTML("<h2>Processes conditions:</h2>"))

              # print("Conditions of the processes:")
              if self.dt_processes_started_at:
                  timedelta = datetime.datetime.now() - self.dt_processes_started_at
                  print("Working for:", timedelta_nice_format(timedelta))
              self.print_table_with_conditions(int_max_processes_to_show=20)
              print(
                  "ALIVE PROCESSES: ",
                  len(self.dict_alive_processes_by_id), "/",
                  len(self.dict_all_processes_by_id))

          @char
          def print_table_with_conditions(self, int_max_processes_to_show=20):
              """"""
              list_list_processes_info = []
              list_headers = ["Process Id", "Status", "Runtime"]
              for process_num in list(self.dict_all_processes_by_id):
                  list_process_info = []
                  process_obj = self.dict_all_processes_by_id[process_num]
                  # Delete if process is not alive
                  if process_num in list(self.dict_alive_processes_by_id):
                      if not process_obj.is_alive():
                          self.dict_alive_processes_by_id.pop(process_num, None)
                  # "Process Id"
                  list_process_info.append(process_num)
                  # "Status"
                  list_process_info.append(process_obj.str_status)
                  # "Runtime"
                  str_runtime = process_obj.get_how_long_this_process_is_running()
                  list_process_info.append(str_runtime)
                  list_list_processes_info.append(list_process_info)
              if len(list_list_processes_info) > int_max_processes_to_show:
                  list_list_additional_columns = []
                  list_list_additional_columns.append(["---", "---", "---"])
                  str_num = str(
                      len(list_list_processes_info) - int_max_processes_to_show)
                  list_list_additional_columns.append(
                      ["Processes", "Hidden", str_num])
                  list_list_additional_columns.append(["---", "---", "---"])
                  list_list_processes_info = \
                      list_list_additional_columns + list_list_processes_info[:20]
              # github  psql  orgtbl  pretty
              print(tabulate(
                  list_list_processes_info, headers=list_headers, tablefmt="pretty"))

          def _delete_all_previous_outputs(self):
              """Delete outputs of previous processes"""
              for str_filename in os.listdir(self.str_dir_for_output):
                  str_file_path = os.path.join(self.str_dir_for_output, str_filename)
                  if not os.path.isfile(str_file_path):
                      continue
                  if "stdout_" in str_filename or "stderr_" in str_filename:
                      try:
                          os.remove(str_file_path)
                      except Exception as ex:
                          print("Cant DELETE previous thread file: ", str_filename)
                          print(ex)

  ### Methods

  `def add_function_to_processing(self, func_to_process, *args, **kwargs)`

    Start running function as process

    ## Args

    `func_to_process` :`function`
      Function to add for processing
    Expand source code

        @char
        def add_function_to_processing(self, func_to_process, *args, **kwargs):
            """Start running function as process

            Args:
                func_to_process (function): Function to add for processing
            """
            new_process = OneProcess(self.str_dir_for_output)
            new_process.start_process(func_to_process, *args, **kwargs)
            if not self.dict_alive_processes_by_id:
                self.dt_processes_started_at = datetime.datetime.now()
            self.dict_all_processes_by_id[new_process.int_process_id] = new_process
            self.dict_alive_processes_by_id[new_process.int_process_id] = new_process

  `def debug_run_of_1_function(self, func_to_process, *args, **kwargs)`
    Expand source code

        @char
        def debug_run_of_1_function(self, func_to_process, *args, **kwargs):
            """"""
            new_process = OneProcess(self.str_dir_for_output)
            new_process.debug_run_of_the_func(func_to_process, *args, **kwargs)
            self.dict_all_processes_by_id[new_process.int_process_id] = new_process
            self.dict_alive_processes_by_id[new_process.int_process_id] = new_process

  `def print_info_about_running_processes(self, int_max_processes_to_show=20)`
    Expand source code

        @char
        def print_info_about_running_processes(self, int_max_processes_to_show=20):
            """"""
            display(HTML("<h2>Processes conditions:</h2>"))

            # print("Conditions of the processes:")
            if self.dt_processes_started_at:
                timedelta = datetime.datetime.now() - self.dt_processes_started_at
                print("Working for:", timedelta_nice_format(timedelta))
            self.print_table_with_conditions(int_max_processes_to_show=20)
            print(
                "ALIVE PROCESSES: ",
                len(self.dict_alive_processes_by_id), "/",
                len(self.dict_all_processes_by_id))

  `def print_table_with_conditions(self, int_max_processes_to_show=20)`
    Expand source code

        @char
        def print_table_with_conditions(self, int_max_processes_to_show=20):
            """"""
            list_list_processes_info = []
            list_headers = ["Process Id", "Status", "Runtime"]
            for process_num in list(self.dict_all_processes_by_id):
                list_process_info = []
                process_obj = self.dict_all_processes_by_id[process_num]
                # Delete if process is not alive
                if process_num in list(self.dict_alive_processes_by_id):
                    if not process_obj.is_alive():
                        self.dict_alive_processes_by_id.pop(process_num, None)
                # "Process Id"
                list_process_info.append(process_num)
                # "Status"
                list_process_info.append(process_obj.str_status)
                # "Runtime"
                str_runtime = process_obj.get_how_long_this_process_is_running()
                list_process_info.append(str_runtime)
                list_list_processes_info.append(list_process_info)
            if len(list_list_processes_info) > int_max_processes_to_show:
                list_list_additional_columns = []
                list_list_additional_columns.append(["---", "---", "---"])
                str_num = str(
                    len(list_list_processes_info) - int_max_processes_to_show)
                list_list_additional_columns.append(
                    ["Processes", "Hidden", str_num])
                list_list_additional_columns.append(["---", "---", "---"])
                list_list_processes_info = \
                    list_list_additional_columns + list_list_processes_info[:20]
            # github  psql  orgtbl  pretty
            print(tabulate(
                list_list_processes_info, headers=list_headers, tablefmt="pretty"))

  `def show_jupyter_widget(self, int_seconds_step=10, int_max_processes_to_show=20)`
    Expand source code

        @char
        def show_jupyter_widget(
                self,
                int_seconds_step=10,
                int_max_processes_to_show=20
        ):
            """"""
            clear_output(wait=True)
            APP_GUI = widget.create_jupyter_widget(self)
            display(APP_GUI)
            with widget.OUTPUT_PROCESSES_CONDITIONS:
                self.print_info_about_running_processes(
                    int_max_processes_to_show=int_max_processes_to_show)
            try:
                while True:
                    if not self.dict_alive_processes_by_id:
                        break
                    for i in range(int_seconds_step):
                        sleep(1)
                        get_ipython().kernel.do_one_iteration()
                    widget.OUTPUT_PROCESSES_CONDITIONS.clear_output(wait=True)
                    with widget.OUTPUT_PROCESSES_CONDITIONS:
                        self.print_info_about_running_processes(
                            int_max_processes_to_show=int_max_processes_to_show)
            except KeyboardInterrupt:
                clear_output(wait=True)
                print("Interrupting running processes")
                self.terminate_all_alive_processes()
                print("---> Done")
            print("All processes were finished")

  `def terminate_all_alive_processes(self)`
    Expand source code

        @char
        def terminate_all_alive_processes(self):
            """"""
            print("Terminating all alive processes")
            for process_num in list(self.dict_alive_processes_by_id):
                print("---> Terminate process: ", process_num + 1)
                process_obj = self.dict_alive_processes_by_id[process_num]
                process_obj.terminate()
                if not process_obj.is_alive():
                    self.dict_alive_processes_by_id.pop(process_num, None)
                    print("------> Done")

  `def wait_till_all_processes_are_over(self, int_seconds_step=10, int_max_processes_to_show=20)`
    Expand source code

        @char
        def wait_till_all_processes_are_over(
                self,
                int_seconds_step=10,
                int_max_processes_to_show=20
        ):
            """"""
            clear_output(wait=True)
            self.print_info_about_running_processes(
                int_max_processes_to_show=int_max_processes_to_show)
            try:
                while True:
                    if not self.dict_alive_processes_by_id:
                        break
                    with yaspin() as sp:
                        for i in range(int_seconds_step):
                            sp.text = "Updating in {} seconds".format(
                                int_seconds_step - i)
                            sleep(1)
                    clear_output(wait=True)
                    self.print_info_about_running_processes(
                        int_max_processes_to_show=int_max_processes_to_show)
            except KeyboardInterrupt:
                clear_output(wait=True)
                print("Interrupting running processes")
                self.terminate_all_alive_processes()
                print("---> Done")
            print("All processes were finished")