#!/usr/bin/env python3
#
# This file is part of Script of Scripts (sos), a workflow system
# for the execution of commands and scripts in different languages.
# Please visit https://github.com/vatlab/SOS for more information.
#
# Copyright (C) 2016 Bo Peng (bpeng@mdanderson.org)
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#

import os
import sys
import re
import time
import shlex
import fnmatch
import contextlib
import subprocess
import argparse
import pkg_resources
import pydoc
import pandas as pd

from io import StringIO
from ipykernel.ipkernel import IPythonKernel
from collections import Sized, defaultdict, OrderedDict

from types import ModuleType
from sos.utils import env, WorkflowDict, short_repr, pretty_size, PrettyRelativeTime, log_to_file
from sos._version import __sos_version__, __version__
from sos.eval import SoS_exec, SoS_eval, interpolate
from sos.syntax import SOS_SECTION_HEADER, SOS_GLOBAL_SECTION_HEADER

from IPython.lib.clipboard import ClipboardEmpty, osx_clipboard_get, tkinter_clipboard_get
from IPython.core.error import UsageError
from IPython.core.display import HTML
from IPython.utils.tokenutil import line_at_cursor, token_at_cursor
from jupyter_client import manager, find_connection_file

from textwrap import dedent

from .completer import SoS_Completer
from .inspector import SoS_Inspector

from .workflow_executor import runfile
from .step_executor import PendingTasks


class FlushableStringIO:
    '''This is a string buffer for output, but it will only
    keep the first 200 lines and the last 10 lines.
    '''

    def __init__(self, kernel, name, *args, **kwargs):
        self.kernel = kernel
        self.name = name

    def write(self, content):
        if content.startswith('!sos_hint:'):
            self.kernel.send_response(self.kernel.iopub_socket, 'display_data',
                                      {
                                          'source': 'SoS',
                                          'metadata': {},
                                          'data': {'text/html': HTML(
                                              f'<div class="sos_hint">{content[10:].strip()}</div>').data}
                                      })
        else:
            if self.name == 'stdout':
                if self.kernel._capture_result is not None:
                    self.kernel._capture_result += content
                if self.kernel._render_result:
                    format_dict, md_dict = self.kernel.format_obj(self.kernel.render_result(content))
                    self.kernel.send_response(self.kernel.iopub_socket, 'display_data',
                                           {'source': 'SoS', 'metadata': md_dict,
                                            'data': format_dict
                                            })
                    return
            self.kernel.send_response(self.kernel.iopub_socket, 'stream',
                                      {'name': self.name, 'text': content})

    def flush(self):
        pass


__all__ = ['SoS_Kernel']


def clipboard_get():
    """ Get text from the clipboard.
    """
    if sys.platform == 'darwin':
        try:
            return osx_clipboard_get()
        except Exception:
            return tkinter_clipboard_get()
    else:
        return tkinter_clipboard_get()


class SoS_Kernel(IPythonKernel):
    implementation = 'SOS'
    implementation_version = __version__
    language = 'sos'
    language_version = __sos_version__
    language_info = {
        'mimetype': 'text/x-sos',
        'name': 'sos',
        'file_extension': '.sos',
        'pygments_lexer': 'sos',
        'codemirror_mode': 'sos',
        'nbconvert_exporter': 'sos_notebook.converter.SoS_Exporter',
    }
    banner = "SoS kernel - script of scripts"

    ALL_MAGICS = {
        'cd',
        'capture',
        'clear',
        'debug',
        'dict',
        'expand',
        'get',
        'matplotlib',
        'paste',
        'preview',
        'pull',
        'push',
        'put',
        'render',
        'rerun',
        'run',
        'save',
        'sandbox',
        'set',
        'sessioninfo',
        'sosrun',
        'sossave',
        'shutdown',
        'taskinfo',
        'tasks',
        'toc',
        'use',
        'with',
    }
    MAGIC_CD = re.compile('^%cd(\s|$)')
    MAGIC_CAPTURE = re.compile('^%capture(\s|$)')
    MAGIC_CLEAR = re.compile('^%clear(\s|$)')
    MAGIC_CONNECT_INFO = re.compile('^%connect_info(\s|$)')
    MAGIC_DEBUG = re.compile('^%debug(\s|$)')
    MAGIC_DICT = re.compile('^%dict(\s|$)')
    MAGIC_EXPAND = re.compile('^%expand(\s|$)')
    MAGIC_GET = re.compile('^%get(\s|$)')
    MAGIC_FRONTEND = re.compile('^%frontend(\s|$)')
    MAGIC_MATPLOTLIB = re.compile('^%matplotlib(\s|$)')
    MAGIC_PASTE = re.compile('^%paste(\s|$)')
    MAGIC_PREVIEW = re.compile('^%preview(\s|$)')
    MAGIC_PULL = re.compile('^%pull(\s|$)')
    MAGIC_PUSH = re.compile('^%push(\s|$)')
    MAGIC_PUT = re.compile('^%put(\s|$)')
    MAGIC_RENDER = re.compile('^%render(\s|$)')
    MAGIC_RERUN = re.compile('^%rerun(\s|$)')
    MAGIC_RUN = re.compile('^%run(\s|$)')
    MAGIC_SAVE = re.compile('^%save(\s|$)')
    MAGIC_SANDBOX = re.compile('^%sandbox(\s|$)')
    MAGIC_SKIP = re.compile('^%skip(\s|$)')
    MAGIC_SET = re.compile('^%set(\s|$)')
    MAGIC_SESSIONINFO = re.compile('^%sessioninfo(\s|$)')
    MAGIC_SOSRUN = re.compile('^%sosrun(\s|$)')
    MAGIC_SOSSAVE = re.compile('^%sossave(\s|$)')
    MAGIC_SHUTDOWN = re.compile('^%shutdown(\s|$)')
    MAGIC_TASKINFO = re.compile('^%taskinfo(\s|$)')
    MAGIC_TASKS = re.compile('^%tasks(\s|$)')
    MAGIC_TOC = re.compile('^%toc(\s|$)')
    MAGIC_USE = re.compile('^%use(\s|$)')
    MAGIC_WITH = re.compile('^%with(\s|$)')

    def get_capture_parser(self):
        parser = argparse.ArgumentParser(prog='%capture',
                                         description='''Capture output (stdout) or output file from a subkernel
                                         as variable in SoS''')
        parser.add_argument('format', default='text', nargs='?', choices=('text', 'json', 'csv', 'tsv'),
                            help='''How to interpret the captured text. By default the captured content will 
                            be saved plain text. Otherwise SoS will try to parse the text as json, csv (comma
                            separated text), tsv (tab separated text), and store text (from text), Pandas DataFrame
                            (from csv or tsv), dict or other types (from json) to the variable.''')
        parser.add_argument('-f', '--from',  dest='__from__', metavar='SOURCE',
                            help='''File from which the content is captured, default to standard output''')
        grp = parser.add_mutually_exclusive_group(required=True)
        grp.add_argument('-t', '--to', dest='__to__', metavar='VAR',
                            help='''Name of variable to which the captured content will be saved''')
        grp.add_argument('-a', '--append', dest='__append__', metavar='VAR',
                            help='''Name of variable to which the captured content will be appended.
                            This option is equivalent to --to if VAR does not exist. If VAR exists
                            and is of the same type of new content (str or dict or DataFrame), the
                            new content will be appended to VAR if VAR is of str (str concatenation),
                            dict (dict update), or DataFrame (DataFrame.append) types. If VAR is of
                            list type, the new content will be appended to the end of the list.''')
        parser.error = self._parse_error
        return parser

    def get_clear_parser(self):
        parser = argparse.ArgumentParser(prog='%clear',
                                         description='''Clear the output of the current cell, or the current
            active cell if executed in the sidepanel.''')
        parser.add_argument('-a', '--all', action='store_true',
                            help='''Clear all output of the current notebook.''')
        parser.add_argument('-s', '--status', nargs='+',
                            help='''Clear tasks that match specifie status (e.g. completed).''')
        parser.error = self._parse_error
        return parser

    def get_debug_parser(self):
        parser = argparse.ArgumentParser(prog='%debug',
                                         description='''Turn on or off debug information''')
        parser.add_argument('status', choices=['on', 'off'],
                            help='''Turn on or off debugging''')
        parser.error = self._parse_error
        return parser

    def get_expand_parser(self):
        parser = argparse.ArgumentParser(prog='%expand',
                                         description='''Expand the script in the current cell with default ({}) or
                specified sigil.''')
        parser.add_argument('sigil', nargs='?', help='''Sigil to be used to interpolated the
            texts. It can be quoted, or be specified as two options.''')
        parser.add_argument('right_sigil', nargs='?', help='''Right sigil if the sigil is
            specified as two pieces.''')
        parser.error = self._parse_error
        return parser

    def get_frontend_parser(self):
        parser = argparse.ArgumentParser(prog='%frontend',
                                         description='''Use specified the subkernel to evaluate current
            cell. soft with tells the start kernel of the cell. If no other
            switch happens the kernel will switch back. However, a %use inside
            the cell will still switch the global kernel. In contrast, a hard
            %with magic will absorb the effect of %use.''')
        parser.add_argument('--list-kernel', action='store_true',
                            help='List kernels')
        parser.add_argument('--default-kernel',
                            help='Default global kernel')
        parser.add_argument('--cell-kernel',
                            help='Kernel to switch to.')
        parser.add_argument('--use-panel', action='store_true',
                            help='If panel is open')
        # pass cell index from notebook so that we know which cell fired
        # the command. Use to set metadata of cell through frontend message
        parser.add_argument('--resume', action='store_true',
                            help='''If the cell is automatically reresumed by frontend, in which
            case -s force should be handled differently.''')
        parser.add_argument('--cell', dest='cell_idx', type=int,
                            help='Index of cell')
        parser.add_argument('--workflow', const='', nargs='?',
                            help='Workflow defined in the notebook')
        parser.add_argument('--filename',
                            help='filename of the current notebook')
        parser.error = self._parse_error
        return parser

    def get_get_parser(self):
        parser = argparse.ArgumentParser(prog='%get',
                                         description='''Get specified variables from another kernel, which is
                by default the SoS kernel.''')
        parser.add_argument('--from', dest='__from__',
                            help='''Name of kernel from which the variables will be obtained.
                Default to the SoS kernel.''')
        parser.add_argument('vars', nargs='*',
                            help='''Names of SoS variables''')
        parser.error = self._parse_error
        return parser

    def get_preview_parser(self):
        parser = argparse.ArgumentParser(prog='%preview',
                                         description='''Preview files, sos variables, or expressions in the
                side panel, or notebook if side panel is not opened, unless
                options --panel or --notebook is specified.''')
        parser.add_argument('items', nargs='*',
                            help='''Filename, variable name, or expression. Wildcard characters
                such as '*' and '?' are allowed for filenames.''')
        parser.add_argument('-k', '--kernel',
                            help='''kernel in which variables will be previewed. By default
            the variable will be previewed in the current kernel of the cell.''')
        parser.add_argument('-w', '--workflow', action='store_true',
                            help='''Preview notebook workflow''')
        parser.add_argument('-o', '--keep-output', action='store_true',
                            help='''Do not clear the output of the side panel.''')
        # this option is currently hidden
        parser.add_argument('-s', '--style', choices=['table', 'scatterplot', 'png'],
                            help='''Option for preview file or variable, which by default is "table"
            for Pandas DataFrame. The %%preview magic also accepts arbitrary additional
            keyword arguments, which would be interpreted by individual style. Passing
            '-h' with '--style' would display the usage information of particular
            style.''')
        parser.add_argument('-r', '--host', dest='host', metavar='HOST', nargs='?', const='',
                            help='''Preview files on specified remote host, which should
            be defined under key host of sos configuration files (preferrably
            in ~/.sos/hosts.yml). If this option is specified without
            value, SoS will list all configured queues and stop.''')
        parser.add_argument('--off', action='store_true',
                            help='''Turn off file preview''')
        loc = parser.add_mutually_exclusive_group()
        loc.add_argument('-p', '--panel', action='store_true',
                         help='''Preview in side panel even if the panel is currently closed''')
        loc.add_argument('-n', '--notebook', action='store_true',
                         help='''Preview in the main notebook.''')
        parser.add_argument('-c', '--config', help='''A configuration file with host
            definitions, in case the definitions are not defined in global or local
            sos config.yml files.''')
        parser.error = self._parse_error
        return parser

    def get_pull_parser(self):
        parser = argparse.ArgumentParser('pull',
                                         description='''Pull files or directories from remote host to local host''')
        parser.add_argument('items', nargs='+', help='''Files or directories to be
            retrieved from remote host. The files should be relative to local file
            system. The files to retrieve are determined by "path_map"
            determined by "paths" definitions of local and remote hosts.''')
        parser.add_argument('-f', '--from', dest='host', nargs='?', const='',
                            help='''Remote host to which the files will be sent. SoS will list all
            configured queues and stop''')
        parser.add_argument('-c', '--config', help='''A configuration file with host
            definitions, in case the definitions are not defined in global or local
            sos config.yml files.''')
        parser.add_argument('-v', '--verbosity', type=int, choices=range(5), default=2,
                            help='''Output error (0), warning (1), info (2), debug (3) and trace (4)
                information to standard output (default to 2).''')
        parser.error = self._parse_error
        return parser

    def get_push_parser(self):
        parser = argparse.ArgumentParser('push',
                                         description='''Push local files or directory to a remote host''')
        parser.add_argument('items', nargs='+', help='''Files or directories to be sent
            to remote host. The location of remote files are determined by "path_map"
            determined by "paths" definitions of local and remote hosts.''')
        parser.add_argument('-t', '--to', dest='host', nargs='?', const='',
                            help='''Remote host to which the files will be sent. SoS will list all
            configured queues if no such key is defined''')
        parser.add_argument('-c', '--config', help='''A configuration file with host
            definitions, in case the definitions are not defined in global or local
            sos config.yml files.''')
        parser.add_argument('-v', '--verbosity', type=int, choices=range(5), default=2,
                            help='''Output error (0), warning (1), info (2), debug (3) and trace (4)
                information to standard output (default to 2).''')
        parser.error = self._parse_error
        return parser

    def get_put_parser(self):
        parser = argparse.ArgumentParser(prog='%put',
                                         description='''Put specified variables in the subkernel to another
            kernel, which is by default the SoS kernel.''')
        parser.add_argument('--to', dest='__to__',
                            help='''Name of kernel from which the variables will be obtained.
                Default to the SoS kernel.''')
        parser.add_argument('vars', nargs='*',
                            help='''Names of SoS variables''')
        parser.error = self._parse_error
        return parser

    def get_render_parser(self):
        parser = argparse.ArgumentParser(prog='%render',
                                         description='''Treat the output of a SoS cell as another format, default to markdown.''')
        parser.add_argument('format', default='Markdown', nargs='?',
                            help='''Format to render output of cell, default to Markdown, but can be any
            format that is supported by the IPython.display module such as HTML, Math, JSON,
            JavaScript and SVG.''')
        parser.error = self._parse_error
        return parser

    def get_rerun_parser(self):
        parser = argparse.ArgumentParser(prog='%rerun',
                                         description='''Re-execute the last executed code, most likely with
            different command line options''')
        parser.error = self._parse_error
        return parser

    def get_run_parser(self):
        parser = argparse.ArgumentParser(prog='%run',
                                         description='''Execute the current cell with specified command line
            arguments. Arguments set by magic %set will be appended at the
            end of command line''')
        parser.error = self._parse_error
        return parser

    def get_save_parser(self):
        parser = argparse.ArgumentParser(prog='%save',
                                         description='''Save the jupyter notebook as workflow (consisting of all sos
            steps defined in cells starting with section header) or a HTML report to
            specified file.''')
        parser.add_argument('filename',
                            help='''Filename of saved report or script.''')
        parser.add_argument('-f', '--force', action='store_true',
                            help='''If destination file already exists, overwrite it.''')
        parser.add_argument('-a', '--append', action='store_true',
                            help='''If destination file already exists, append to it.''')
        parser.add_argument('-x', '--set-executable', dest="setx", action='store_true',
                            help='''Set `executable` permission to saved script.''')
        parser.error = self._parse_error
        return parser

    def get_set_parser(self):
        parser = argparse.ArgumentParser(prog='%set',
                                         description='''Set persistent command line options for SoS runs.''')
        parser.error = self._parse_error
        return parser

    def get_sessioninfo_parser(self):
        parser = argparse.ArgumentParser(prog='%sessioninfo',
                                         description='''List the session info of all subkernels, and information
            stored in variable sessioninfo''')
        parser.error = self._parse_error
        return parser

    def get_shutdown_parser(self):
        parser = argparse.ArgumentParser(prog='%shutdown',
                                         description='''Shutdown or restart specified subkernel''')
        parser.add_argument('kernel', nargs='?',
                            help='''Name of the kernel to be restarted, default to the
            current running kernel.''')
        parser.add_argument('-r', '--restart', action='store_true',
                            help='''Restart the kernel''')
        parser.error = self._parse_error
        return parser

    def get_sosrun_parser(self):
        parser = argparse.ArgumentParser(prog='%sosrun',
                                         description='''Execute the entire notebook with steps consisting of SoS
            cells (cells with SoS kernel) with section header, with specified command
            line arguments. Arguments set by magic %set will be appended at the
            end of command line''')
        parser.error = self._parse_error
        return parser

    def get_sossave_parser(self):
        parser = argparse.ArgumentParser(prog='%sossave',
                                         description='''Save the jupyter notebook as workflow (consisting of all sos
            steps defined in cells starting with section header) or a HTML report to
            specified file.''')
        parser.add_argument('filename', nargs='?',
                            help='''Filename of saved report or script. Default to notebookname with file
            extension determined by option --to.''')
        parser.add_argument('-t', '--to', dest='__to__', choices=['sos', 'html'],
                            help='''Destination format, default to sos.''')
        parser.add_argument('-c', '--commit', action='store_true',
                            help='''Commit the saved file to git directory using command
            git commit FILE''')
        parser.add_argument('-a', '--all', action='store_true',
                            help='''The --all option for sos convert script.ipynb script.sos, which
            saves all cells and their metadata to a .sos file, that contains all input
            information of the notebook but might not be executable in batch mode.''')
        parser.add_argument('-m', '--message',
                            help='''Message for git commit. Default to "save FILENAME"''')
        parser.add_argument('-p', '--push', action='store_true',
                            help='''Push the commit with command "git push"''')
        parser.add_argument('-f', '--force', action='store_true',
                            help='''If destination file already exists, overwrite it.''')
        parser.add_argument('-x', '--set-executable', dest="setx", action='store_true',
                            help='''Set `executable` permission to saved script.''')
        parser.add_argument('--template', default='default-sos-template',
                            help='''Template to generate HTML output. The default template is a
            template defined by configuration key default-sos-template, or
            sos-report if such a key does not exist.''')
        parser.error = self._parse_error
        return parser

    def get_taskinfo_parser(self):
        parser = argparse.ArgumentParser(prog='%taskinfo',
                                         description='''Get information on specified task. By default
            sos would query against all running task queues but it would
            start a task queue and query status if option -q is specified.
            ''')
        parser.add_argument('task', help='ID of task')
        parser.add_argument('-q', '--queue',
                            help='''Task queue on which the task is executed.''')
        parser.add_argument('-c', '--config', help='''A configuration file with host
            definitions, in case the definitions are not defined in global or local
            sos config.yml files.''')
        parser.error = self._parse_error
        return parser

    def get_tasks_parser(self):
        parser = argparse.ArgumentParser(prog='%tasks',
                                         description='''Show a list of tasks from specified queue''')
        parser.add_argument('tasks', nargs='*', help='ID of tasks')
        parser.add_argument('-s', '--status', nargs='*',
                            help='''Display tasks of specified status. Default to all.''')
        parser.add_argument('-q', '--queue',
                            help='''Task queue on which the tasks are retrived.''')
        parser.add_argument('--age', help='''Limit to tasks that is created more than
            (default) or within specified age. Value of this parameter can be in units
            s (second), m (minute), h (hour), or d (day, default), with optional
            prefix + for older (default) and - for younder than specified age.''')
        parser.add_argument('-c', '--config', help='''A configuration file with host
            definitions, in case the definitions are not defined in global or local
            sos config.yml files.''')
        parser.error = self._parse_error
        return parser

    def get_toc_parser(self):
        parser = argparse.ArgumentParser(prog='%toc',
                                         description='''Display toc in the side panel''')
        parser.error = self._parse_error
        return parser

    def get_use_parser(self):
        parser = argparse.ArgumentParser(prog='%use',
                                         description='''Switch to a specified subkernel.''')
        parser.add_argument('name', nargs='?', default='',
                            help='''Displayed name of kernel to start (if no kernel with name is
            specified) or switch to (if a kernel with this name is already started).
            The name is usually a kernel name (e.g. %%use ir) or a language name
            (e.g. %%use R) in which case the language name will be used. One or
            more parameters --language or --kernel will need to be specified
            if a new name is used to start a separate instance of a kernel.''')
        parser.add_argument('-k', '--kernel',
                            help='''kernel name as displayed in the output of jupyter kernelspec
            list. Default to the default kernel of selected language (e.g. ir for
            language R.''')
        parser.add_argument('-l', '--language',
                            help='''Language extension that enables magics such as %%get and %%put
            for the kernel, which should be in the name of a registered language
            (e.g. R), or a specific language module in the format of
            package.module:class. SoS maitains a list of languages and kernels
            so this option is only needed for starting a new instance of a kernel.
            ''')
        parser.add_argument('-c', '--color',
                            help='''Background color of new or existing kernel, which overrides
            the default color of the language. A special value "default" can be
            used to reset color to default.''')
        parser.add_argument('-r', '--restart', action='store_true',
                            help='''Restart the kernel if it is running.''')
        parser.add_argument('-i', '--in', nargs='*', dest='in_vars',
                            help='Input variables (variables to get from SoS kernel)')
        parser.add_argument('-o', '--out', nargs='*', dest='out_vars',
                            help='''Output variables (variables to put back to SoS kernel
            before switching back to the SoS kernel''')
        parser.error = self._parse_error
        return parser

    def get_with_parser(self):
        parser = argparse.ArgumentParser(prog='%with',
                                         description='''Use specified the subkernel to evaluate current
            cell''')
        parser.add_argument('name', nargs='?', default='',
                            help='''Displayed name of kernel to start (if no kernel with name is
            specified) or switch to (if a kernel with this name is already started).
            The name is usually a kernel name (e.g. %use ir) or a language name
            (e.g. %use R) in which case the language name will be used. One or
            more parameters --language or --kernel will need to be specified
            if a new name is used to start a separate instance of a kernel.''')
        parser.add_argument('-k', '--kernel',
                            help='''kernel name as displayed in the output of jupyter kernelspec
            list. Default to the default kernel of selected language (e.g. ir for
            language R.''')
        parser.add_argument('-l', '--language',
                            help='''Language extension that enables magics such as %get and %put
            for the kernel, which should be in the name of a registered language
            (e.g. R), or a specific language module in the format of
            package.module:class. SoS maitains a list of languages and kernels
            so this option is only needed for starting a new instance of a kernel.
            ''')
        parser.add_argument('-c', '--color',
                            help='''Background color of existing or new kernel, which overrides
            the default color of the language. A special value "default" can be
            used to reset color to default.''')
        parser.add_argument('-r', '--restart', action='store_true',
                            help='''Restart the kernel if it is running.''')
        parser.add_argument('-i', '--in', nargs='*', dest='in_vars',
                            help='Input variables (variables to get from SoS kernel)')
        parser.add_argument('-o', '--out', nargs='*', dest='out_vars',
                            help='''Output variables (variables to put back to SoS kernel
            before switching back to the SoS kernel''')
        parser.error = self._parse_error
        return parser

    def find_kernel(self, name, kernel=None, language=None, color=None, notify_frontend=True):
        # find from subkernel name
        def update_existing(idx):
            x = self._kernel_list[idx]
            if (kernel is not None and kernel != x[1]) or (language is not None and language != x[2]):
                raise ValueError(f'Cannot change kernel or language of predefined subkernel {name}')
            if color is not None:
                if color == 'default':
                    if self._kernel_list[idx][2]:
                        self._kernel_list[idx][3] = self._supported_languages[
                            self._kernel_list[idx][2]].background_color
                    else:
                        self._kernel_list[idx][3] = ''
                else:
                    self._kernel_list[idx][3] = color
                if notify_frontend:
                    self.send_frontend_msg('kernel-list', self.get_kernel_list())

        # if the language module cannot be loaded for some reason
        if name in self._failed_languages:
            raise self._failed_languages[name]
        # find from language name (subkernel name, which is usually language name)
        for idx, x in enumerate(self.get_kernel_list()):
            if x[0] == name:
                if x[0] == 'SoS' or x[2] or language is None:
                    update_existing(idx)
                    return x
                else:
                    if not kernel:
                        kernel = name
                    break
        # find from kernel name
        for idx, x in enumerate(self._kernel_list):
            if x[1] == name:
                # if exist language or no new language defined.
                if x[2] or language is None:
                    update_existing(idx)
                    return x
                else:
                    # otherwise, try to use the new language
                    kernel = name
                    break

        # now, no kernel is found, name has to be a new name and we need some definition
        # if kernel is defined
        def add_or_replace(kdef):
            for idx, x in enumerate(self._kernel_list):
                if x[0] == kdef[0]:
                    self._kernel_list[idx] = kdef
                    return self._kernel_list[idx]
                else:
                    self._kernel_list.append(kdef)
                    return self._kernel_list[-1]

        if kernel is not None:
            # in this case kernel should have been defined in kernel list
            if kernel not in [x[1] for x in self._kernel_list]:
                raise ValueError(
                    f'Unrecognized Jupyter kernel name {kernel}. Please make sure it is properly installed and appear in the output of command "jupyter kenelspec list"')
            # now this a new instance for an existing kernel
            kdef = [x for x in self._kernel_list if x[1] == kernel][0]
            if not language:
                if color == 'default':
                    if kdef[2]:
                        color = self._supported_languages[kdef[2]].background_color
                    else:
                        color = kdef[3]
                new_def = add_or_replace([name, kdef[1], kdef[2], kdef[3] if color is None else color,
                                          getattr(self._supported_languages[kdef[2]], 'options', {}) if kdef[
                                              2] else {}])
                if notify_frontend:
                    self.send_frontend_msg('kernel-list', self.get_kernel_list())
                return new_def
            else:
                # if language is defined,
                if ':' in language:
                    # if this is a new module, let us create an entry point and load
                    from pkg_resources import EntryPoint
                    mn, attr = language.split(':', 1)
                    ep = EntryPoint(name=kernel, module_name=mn, attrs=tuple(attr.split('.')))
                    try:
                        plugin = ep.resolve()
                        self._supported_languages[name] = plugin
                        # for convenience, we create two entries for, e.g. R and ir
                        # but only if there is no existing definition
                        for supported_lan, supported_kernels in plugin.supported_kernels.items():
                            for supported_kernel in supported_kernels:
                                if name != supported_kernel and supported_kernel not in self._supported_languages:
                                    self._supported_languages[supported_kernel] = plugin
                            if supported_lan not in self._supported_languages:
                                self._supported_languages[supported_lan] = plugin
                    except Exception as e:
                        raise RuntimeError(f'Failed to load language {language}: {e}')
                    #
                    if color == 'default':
                        color = plugin.background_color
                    new_def = add_or_replace([name, kdef[1], kernel, kdef[3] if color is None else color,
                                              getattr(plugin, 'options', {})])
                else:
                    # if should be defined ...
                    if language not in self._supported_languages:
                        raise RuntimeError(
                            f'Unrecognized language definition {language}, which should be a known language name or a class in the format of package.module:class')
                    #
                    self._supported_languages[name] = self._supported_languages[language]
                    if color == 'default':
                        color = self._supported_languages[name].background_color
                    new_def = add_or_replace([name, kdef[1], language, kdef[3] if color is None else color,
                                              getattr(self._supported_languages[name], 'options', {})])
                if notify_frontend:
                    self.send_frontend_msg('kernel-list', self.get_kernel_list())
                return new_def
        elif language is not None:
            # kernel is not defined and we only have language
            if ':' in language:
                # if this is a new module, let us create an entry point and load
                from pkg_resources import EntryPoint
                mn, attr = language.split(':', 1)
                ep = EntryPoint(name='__unknown__', module_name=mn, attrs=tuple(attr.split('.')))
                try:
                    plugin = ep.resolve()
                    self._supported_languages[name] = plugin
                except Exception as e:
                    raise RuntimeError(f'Failed to load language {language}: {e}')
                if name in plugin.supported_kernels:
                    # if name is defined in the module, only search kernels for this language
                    avail_kernels = [x for x in plugin.supported_kernels[name] if
                                     x in [y[1] for y in self._kernel_list]]
                else:
                    # otherwise we search all supported kernels
                    avail_kernels = [x for x in sum(plugin.supported_kernels.values(), []) if
                                     x in [y[1] for y in self._kernel_list]]

                if not avail_kernels:
                    raise ValueError(
                        'Failed to find any of the kernels {} supported by language {}. Please make sure it is properly installed and appear in the output of command "jupyter kenelspec list"'.format(
                            ', '.join(sum(plugin.supported_kernels.values(), [])), language))
                # use the first available kernel
                if color == 'default':
                    color = plugin.background_color
                # find the language that has the kernel
                lan_name = list({x: y for x, y in plugin.supported_kernels.items() if avail_kernels[0] in y}.keys())[0]
                new_def = add_or_replace(
                    [name, avail_kernels[0], lan_name, plugin.background_color if color is None else color,
                     getattr(plugin, 'options', {})])
            else:
                # if a language name is specified (not a path to module), if should be defined in setup.py
                if language not in self._supported_languages:
                    raise RuntimeError(f'Unrecognized language definition {language}')
                #
                plugin = self._supported_languages[language]
                if language in plugin.supported_kernels:
                    avail_kernels = [x for x in plugin.supported_kernels[language] if
                                     x in [y[1] for y in self._kernel_list]]
                else:
                    avail_kernels = [x for x in sum(plugin.supported_kernels.values(), []) if
                                     x in [y[1] for y in self._kernel_list]]
                if not avail_kernels:
                    raise ValueError(
                        'Failed to find any of the kernels {} supported by language {}. Please make sure it is properly installed and appear in the output of command "jupyter kenelspec list"'.format(
                            ', '.join(sum(self._supported_languages[language].supported_kernels.values(), [])),
                            language))

                new_def = add_or_replace([
                    name, avail_kernels[0], language,
                    self._supported_languages[
                        language].background_color if color is None or color == 'default' else color,
                    getattr(self._supported_languages[language], 'options', {})])

            self.send_frontend_msg('kernel-list', self.get_kernel_list())
            return new_def
        else:
            # let us check if there is something wrong with the pre-defined language
            for entrypoint in pkg_resources.iter_entry_points(group='sos_languages'):
                if entrypoint.name == name:
                    # there must be something wrong, let us trigger the exception here
                    entrypoint.load()
            # if nothing is triggerred, kernel is not defined, return a general message
            raise ValueError(
                f'No subkernel named {name} is found. Please make sure that you have the kernel installed (listed in the output of "jupyter kernelspec list" and usable in jupyter by itself), install appropriate language module (e.g. "pip install sos-r"), restart jupyter notebook and try again.')

    def get_supported_languages(self):
        if self._supported_languages is not None:
            return self._supported_languages
        group = 'sos_languages'
        self._supported_languages = {}
        for entrypoint in pkg_resources.iter_entry_points(group=group):
            # Grab the function that is the actual plugin.
            name = entrypoint.name
            try:
                plugin = entrypoint.load()
                self._supported_languages[name] = plugin
            except Exception as e:
                self._failed_languages[name] = e
        return self._supported_languages

    supported_languages = property(lambda self: self.get_supported_languages())

    def get_completer(self):
        if self._completer is None:
            self._completer = SoS_Completer(self)
        return self._completer

    completer = property(lambda self: self.get_completer())

    def get_inspector(self):
        if self._inspector is None:
            self._inspector = SoS_Inspector(self)
        return self._inspector

    inspector = property(lambda self: self.get_inspector())

    def __init__(self, **kwargs):
        super(SoS_Kernel, self).__init__(**kwargs)
        self.options = ''
        self.kernel = 'SoS'
        # a dictionary of started kernels, with the format of
        #
        # 'R': ['ir', 'sos.R.sos_R', '#FFEEAABB']
        #
        # Note that:
        #
        # 'R' is the displayed name of the kernel.
        # 'ir' is the kernel name.
        # 'sos.R.sos_R' is the language module.
        # '#FFEEAABB' is the background color
        #
        self.kernels = {}
        # self.shell = InteractiveShell.instance()
        self.format_obj = self.shell.display_formatter.format

        self.previewers = None
        self.original_keys = None
        self._supported_languages = None
        self._completer = None
        self._inspector = None
        self._real_execution_count = 1
        self._execution_count = 1
        self._debug_mode = False
        self._use_panel = None
        self._resume_execution = False

        self.frontend_comm = None
        self.comm_manager.register_target('sos_comm', self.sos_comm)
        self.cell_idx = None
        self.my_tasks = {}
        self.last_executed_code = ''
        self.RET_VARS = []
        self._workflow = ''
        #
        self._workflow_mode = False
        self._render_result = False
        self._capture_result = None
        self._failed_languages = {}
        env.__task_notifier__ = self.notify_task_status

    def handle_taskinfo(self, task_id, task_queue, side_panel=None):
        # requesting information on task
        from sos.hosts import Host
        host = Host(task_queue)
        result = host._task_engine.query_tasks([task_id], verbosity=2, html=True)
        # log_to_file(result)
        if side_panel is True:
            self.send_frontend_msg('display_data',
                                   {'metadata': {},
                                    'data': {'text/plain': result,
                                             'text/html': HTML(result).data
                                             }})
        else:
            self.send_response(self.iopub_socket, 'display_data',
                               {'metadata': {},
                                'data': {'text/plain': result,
                                         'text/html': HTML(result).data
                                         }})
        # now, there is a possibility that the status of the task is different from what
        # task engine knows (e.g. a task is rerun outside of jupyter). In this case, since we
        # already get the status, we should update the task engine...
        #
        # <tr><th align="right"  width="30%">Status</th><td align="left"><div class="one_liner">completed</div></td></tr>
        status = result.split('>Status<', 1)[-1].split('</div', 1)[0].split('>')[-1]
        host._task_engine.update_task_status(task_id, status)

    def handle_tasks(self, tasks, queue='localhost', status=None, age=None):
        from sos.hosts import Host
        try:
            host = Host(queue)
        except Exception as e:
            self.warn('Invalid task queu {}: {}'.format(queue, e))
            return
        # get all tasks
        for tid, tst, tdt in host._task_engine.monitor_tasks(tasks, status=status, age=age):
            self.notify_task_status(['new-status', queue, tid, tst, tdt])
        self.send_frontend_msg('update-duration', {})

    def handle_sessioninfo(self):
        #
        from sos.utils import loaded_modules
        result = OrderedDict()
        #
        result['SoS'] = [('SoS Version', __version__)]
        result['SoS'].extend(loaded_modules(env.sos_dict))
        #
        cur_kernel = self.kernel
        try:
            for kernel in self.kernels.keys():
                kinfo = self.find_kernel(kernel)
                self.switch_kernel(kernel)
                result[kernel] = [
                    ('Kernel', kinfo[1]),
                    ('Language', kinfo[2])
                ]
                if kernel not in self.supported_languages:
                    continue
                lan = self.supported_languages[kernel]
                if hasattr(lan, 'sessioninfo'):
                    try:
                        sinfo = lan(self, kinfo[1]).sessioninfo()
                        if isinstance(sinfo, str):
                            result[kernel].append([sinfo])
                        elif isinstance(sinfo, dict):
                            result[kernel].extend(list(sinfo.items()))
                        elif isinstance(sinfo, list):
                            result[kernel].extend(sinfo)
                        else:
                            self.warn(f'Unrecognized session info: {sinfo}')
                    except Exception as e:
                        self.warn(f'Failed to obtain sessioninfo of kernel {kernel}: {e}')
        finally:
            self.switch_kernel(cur_kernel)
        #
        if 'sessioninfo' in env.sos_dict:
            result.update(env.sos_dict['sessioninfo'])
        #
        res = ''
        for key, items in result.items():
            res += f'<p class="session_section">{key}</p>\n'
            res += '<table class="session_info">\n'
            for item in items:
                res += '<tr>\n'
                if isinstance(item, str):
                    res += f'<td colspan="2"><pre>{item}</pre></td>\n'
                elif len(item) == 1:
                    res += f'<td colspan="2"><pre>{item[0]}</pre></td>\n'
                elif len(item) == 2:
                    res += f'<th>{item[0]}</th><td><pre>{item[1]}</pre></td>\n'
                else:
                    self.warn(f'Invalid session info item of type {item.__class__.__name__}: {short_repr(item)}')
                res += '</tr>\n'
            res += '</table>\n'
        self.send_response(self.iopub_socket, 'display_data',
                           {'source': 'SoS', 'metadata': {},
                            'data': {'text/html': HTML(res).data}})

    def sos_comm(self, comm, msg):
        # record frontend_comm to send messages
        self.frontend_comm = comm

        @comm.on_msg
        def handle_frontend_msg(msg):
            content = msg['content']['data']
            # log_to_file(msg)
            for k, v in content.items():
                if k == 'list-kernel':
                    self.send_frontend_msg('kernel-list', self.get_kernel_list(v))
                elif k == 'kill-task':
                    # kill specified task
                    from sos.hosts import Host
                    Host(v[1])._task_engine.kill_tasks([v[0]])
                    self.notify_task_status(['change-status', v[1], v[0], 'aborted'])
                elif k == 'resume-task':
                    # kill specified task
                    from sos.hosts import Host
                    Host(v[1])._task_engine.resume_task(v[0])
                    self.notify_task_status(['change-status', v[1], v[0], 'pending'])
                elif k == 'task-info':
                    self.handle_taskinfo(v[0], v[1], side_panel=True)
                elif k == 'workflow':
                    self._workflow = '#!/usr/bin/env sos-runner\n#fileformat=SOS1.0\n\n' + v
                elif k == 'update-task-status':
                    if not isinstance(v, list):
                        continue
                    # split by host ...
                    host_status = defaultdict(list)
                    for name in v:
                        if not name.startswith('status_'):
                            continue
                        try:
                            tqu, tid = name[7:].rsplit('_', 1)
                        except Exception:
                            # incorrect ID...
                            continue
                        host_status[tqu].append(tid)
                    # log_to_file(host_status)
                    #
                    from sos.hosts import Host
                    for tqu, tids in host_status.items():
                        try:
                            h = Host(tqu)
                        except Exception:
                            continue
                        for tid in tids:
                            tst = h._task_engine.check_task_status(tid, unknown='unknown')
                            self.notify_task_status(['change-status', tqu, tid, tst])
                    self.send_frontend_msg('update-duration', {})
                elif k == 'paste-table':
                    try:
                        from tabulate import tabulate
                        df = pd.read_clipboard()
                        tbl = tabulate(df, headers='keys', tablefmt='pipe')
                        self.send_frontend_msg('paste-table', tbl)
                        if self._debug_mode:
                            log_to_file(tbl)
                    except Exception as e:
                        self.send_frontend_msg('alert', f'Failed to paste clipboard as table: {e}')
                else:
                    # this somehow does not work
                    self.warn(f'Unknown message {k}: {v}')

    status_class = {
        'pending': 'fa-square-o',
        'submitted': 'fa-spinner',
        'running': 'fa-spinner fa-pulse fa-spin',
        'result-ready': 'fa-files-o',
        'completed': 'fa-check-square-o',
        'failed': 'fa-times-circle-o',
        'aborted': 'fa-frown-o',
        'signature-mismatch': 'fa-question',
        'unknown': 'fa-question',
    }

    def notify_task_status(self, task_status):
        action_class = {
            'pending': 'fa-stop',
            'submitted': 'fa-stop',
            'running': 'fa-stop',
            'result-ready': 'fa-play',
            'completed': 'fa-play',
            'failed': 'fa-play',
            'aborted': 'fa-play',
            'signature-mismatch': 'fa-play',
            'unknown': 'fa-question',
        }

        action_func = {
            'pending': 'kill_task',
            'submitted': 'kill_task',
            'running': 'kill_task',
            'result-ready': 'resume_task',
            'completed': 'resume_task',
            'failed': 'resume_task',
            'aborted': 'resume_task',
            'signature-mismatch': 'resume_task',
            'unknown': 'function(){}',
        }

        if task_status[0] == 'new-status':
            tqu, tid, tst, tdt = task_status[1:]
            self.send_response(self.iopub_socket, 'display_data',
                               {
                                   'source': 'SoS',
                                   'metadata': {},
                                   'data': {'text/html':
                                                HTML(f'''<table id="table_{tqu}_{tid}" class="task_table"><tr style="border: 0px">
                        <td style="border: 0px">
                        <i id="status_{tqu}_{tid}"
                            class="fa fa-2x fa-fw {self.status_class[tst]}"
                            onmouseover="$('#status_{tqu}_{tid}').addClass('{action_class[tst]} task_hover').removeClass('{self.status_class[tst]}')"
                            onmouseleave="$('#status_{tqu}_{tid}').addClass('{self.status_class[tst]}').removeClass('{action_class[tst]} task_hover')"
                            onclick="{action_func[tst]}('{tid}', '{tqu}')"
                        ></i> </td>
                        <td style="border:0px"><a onclick="task_info('{tid}', '{tqu}')"><pre>{tid}</pre></a></td>
                        <td style="border:0px">&nbsp;</td>
                        <td style="border:0px;text-align=right;">
                        <pre><time id="duration_{tqu}_{tid}" class="{tst}" datetime="{tdt*1000}">{PrettyRelativeTime(time.time() - tdt)}</time></pre></td>
                        </tr>
                        </table>''').data
                                            }
                               })
            # keep tracks of my tasks to avoid updating status of
            # tasks that does not belong to the notebook
            self.my_tasks[(tqu, tid)] = time.time()
        elif task_status[0] == 'remove-task':
            tqu, tid = task_status[1:]
            if (tqu, tid) in self.my_tasks:
                self.send_frontend_msg('remove-task', [tqu, tid])
        elif task_status[0] == 'change-status':
            tqu, tid, tst = task_status[1:]
            if tst not in ('pending', 'submitted', 'running', 'result-ready', 'completed',
                           'failed', 'aborted', 'signature-mismatch'):
                tst = 'unknown'
            self.send_frontend_msg('task-status',
                                   [tqu, tid, tst, self.status_class[tst], action_class[tst], action_func[tst]])
            self.my_tasks[(tqu, tid)] = time.time()
        elif task_status[0] == 'pulse-status':
            tqu, tid, tst = task_status[1:]
            if tst not in ('pending', 'submitted', 'running', 'result-ready', 'completed',
                           'failed', 'aborted', 'signature-mismatch'):
                tst = 'unknown'
            if (tqu, tid) in self.my_tasks:
                if time.time() - self.my_tasks[(tqu, tid)] < 20:
                    # if it has been within the first 20 seconds of new or updated message
                    # can confirm to verify it has been successfully delivered. Otherwise
                    # ignore such message
                    self.send_frontend_msg('task-status',
                                           [tqu, tid, tst, self.status_class[tst], action_class[tst], action_func[tst]])
            else:
                # perhaps the pulse one does not have an initial value yet
                self.send_frontend_msg('task-status',
                                   [tqu, tid, tst, self.status_class[tst], action_class[tst], action_func[tst]])
                self.my_tasks[(tqu, tid)] = time.time()
        else:
            raise RuntimeError(f'Unrecognized status change message {task_status}')

    def send_frontend_msg(self, msg_type, msg=None):
        # if comm is never created by frontend, the kernel is in test mode without frontend
        if self._use_panel is False and msg_type in ('display_data', 'stream', 'preview-input'):
            if msg_type in ('display_data', 'stream'):
                self.send_response(self.iopub_socket, msg_type, {} if msg is None else msg)
            elif msg_type == 'preview-input':
                self.send_response(self.iopub_socket, 'display_data',
                                   {
                                       'source': 'SoS',
                                       'metadata': {},
                                       'data': {'text/html': HTML(f'<div class="sos_hint">{msg}</div>').data}
                                   })
        elif self.frontend_comm:
            self.frontend_comm.send({} if msg is None else msg, {'msg_type': msg_type})
        else:
            self.warn('Frontend communicator is broken. Please restart jupyter server')

    def _reset_dict(self):
        env.sos_dict = WorkflowDict()
        SoS_exec('import os, sys, glob', None)
        SoS_exec('from sos.runtime import *', None)
        SoS_exec("run_mode = 'interactive'", None)
        self.original_keys = set(env.sos_dict._dict.keys()) | {'SOS_VERSION', 'CONFIG', \
                                                               'step_name', '__builtins__', 'input', 'output',
                                                               'depends'}

    @contextlib.contextmanager
    def redirect_sos_io(self):
        save_stdout = sys.stdout
        save_stderr = sys.stderr
        sys.stdout = FlushableStringIO(self, 'stdout')
        sys.stderr = FlushableStringIO(self, 'stderr')
        yield
        sys.stdout = save_stdout
        sys.stderr = save_stderr

    def do_is_complete(self, code):
        '''check if new line is in order'''
        code = code.strip()
        if not code:
            return {'status': 'complete', 'indent': ''}
        if any(code.startswith(x) for x in ['%dict', '%paste', '%edit', '%cd', '!']):
            return {'status': 'complete', 'indent': ''}
        if code.endswith(':') or code.endswith(','):
            return {'status': 'incomplete', 'indent': '  '}
        lines = code.split('\n')
        if lines[-1].startswith(' ') or lines[-1].startswith('\t'):
            # if it is a new line, complte
            empty = [idx for idx, x in enumerate(lines[-1]) if x not in (' ', '\t')][0]
            return {'status': 'incomplete', 'indent': lines[-1][:empty]}
        #
        if SOS_SECTION_HEADER.match(lines[-1]):
            return {'status': 'incomplete', 'indent': ''}
        #
        return {'status': 'incomplete', 'indent': ''}

    def do_inspect(self, code, cursor_pos, detail_level=0):
        line, offset = line_at_cursor(code, cursor_pos)
        name = token_at_cursor(code, cursor_pos)
        data = self.inspector.inspect(name, line, cursor_pos - offset)

        reply_content = {'status': 'ok'}
        reply_content['metadata'] = {}
        reply_content['found'] = True if data else False
        reply_content['data'] = data
        return reply_content

    def do_complete(self, code, cursor_pos):
        text, matches = self.completer.complete_text(code, cursor_pos)
        return {'matches': matches,
                'cursor_end': cursor_pos,
                'cursor_start': cursor_pos - len(text),
                'metadata': {},
                'status': 'ok'}

    def warn(self, message):
        message = str(message).rstrip() + '\n'
        if message.strip():
            self.send_response(self.iopub_socket, 'stream',
                               {'name': 'stderr', 'text': message})

    def get_magic_and_code(self, code, warn_remaining=False):
        if code.startswith('%') or code.startswith('!'):
            lines = re.split(r'(?<!\\)\n', code, 1)
            # remove lines joint by \
            lines[0] = lines[0].replace('\\\n', '')
        else:
            lines = code.split('\n', 1)

        pieces = self._interpolate_text(lines[0], quiet=False).strip().split(None, 1)
        if len(pieces) == 2:
            command_line = pieces[1]
        else:
            command_line = ''
        remaining_code = lines[1] if len(lines) > 1 else ''
        if warn_remaining and remaining_code.strip():
            self.warn('Statement {} ignored'.format(short_repr(remaining_code)))
        return command_line, remaining_code

    def run_cell(self, code, silent, store_history, on_error=None):
        #
        if not self.KM.is_alive():
            self.send_response(self.iopub_socket, 'stream',
                               dict(name='stdout', text='Restarting kernel "{}"\n'.format(self.kernel)))
            self.KM.restart_kernel(now=False)
            self.KC = self.KM.client()
        # flush stale replies, which could have been ignored, due to missed heartbeats
        while self.KC.shell_channel.msg_ready():
            self.KC.shell_channel.get_msg()
        # executing code in another kernel
        self.KC.execute(code, silent=silent, store_history=store_history)

        # first thing is wait for any side effects (output, stdin, etc.)
        _execution_state = "busy"
        while _execution_state != 'idle':
            # display intermediate print statements, etc.
            while self.KC.iopub_channel.msg_ready():
                sub_msg = self.KC.iopub_channel.get_msg()
                msg_type = sub_msg['header']['msg_type']
                if self._debug_mode:
                    log_to_file(f'MSG TYPE {msg_type}')
                    log_to_file(f'CONTENT  {sub_msg["content"]}')
                if msg_type == 'status':
                    _execution_state = sub_msg["content"]["execution_state"]
                else:
                    if msg_type in ('execute_input', 'execute_result'):
                        # override execution count with the master count,
                        # not sure if it is needed
                        sub_msg['content']['execution_count'] = self._execution_count
                    #
                    if silent and msg_type in ['display_data', 'stream', 'execute_result']:
                        continue
                    # NOTE: we do not send status of sub kernel alone because
                    # these are generated automatically during the execution of
                    # "this cell" in SoS kernel
                    if (self._capture_result is not None or self._render_result) and msg_type == 'stream' and sub_msg['content']['name'] == 'stdout':
                        if self._capture_result is not None:
                            self._capture_result += sub_msg['content']['text']
                            self.send_response(self.iopub_socket, 'stream',
                                    {'name': 'stdout', 'text': sub_msg['content']['text']})
                        else:
                            format_dict, md_dict = self.format_obj(self.render_result(sub_msg['content']['text']))
                            self.send_response(self.iopub_socket, 'display_data',
                                           {'source': 'SoS', 'metadata': md_dict,
                                            'data': format_dict
                                            })
                    elif msg_type == 'error':
                        if on_error and not self._debug_mode:
                            self.warn(on_error)
                        else:
                            self.send_response(self.iopub_socket, msg_type, sub_msg['content'])
                    elif msg_type == 'stream' and sub_msg['content']['name'] == 'stderr':
                        self.warn(sub_msg['content']['text'])
                    else:
                        self.send_response(self.iopub_socket, msg_type, sub_msg['content'])
        #
        # now get the real result
        reply = self.KC.get_shell_msg(timeout=10)
        reply['content']['execution_count'] = self._execution_count
        return reply['content']

    def switch_kernel(self, kernel, in_vars=None, ret_vars=None, kernel_name=None, language=None, color=None):
        # switching to a non-sos kernel
        if not kernel:
            kinfo = self.find_kernel(self.kernel)
            self.send_response(self.iopub_socket, 'stream',
                               dict(name='stdout', text='''\
Current subkernel: {} (kernel={}, language={}, color="{}")
Active subkernels: {}
Available subkernels:\n{}'''.format(
                                   kinfo[0], kinfo[1], kinfo[2] if kinfo[2] else "undefined", kinfo[3],
                                   ', '.join(self.kernels.keys()),
                                   '\n'.join(['    {} ({})'.format(x[0], x[1]) for x in self.get_kernel_list()]))))
            return
        kinfo = self.find_kernel(kernel, kernel_name, language, color)
        if kinfo[0] == self.kernel:
            # the same kernel, do nothing?
            # but the senario can be
            #
            # kernel in SoS
            # cell R
            # %use R -i n
            #
            # SoS get:
            #
            # %softwidth --default-kernel R --cell-kernel R
            # %use R -i n
            #
            # Now, SoS -> R without variable passing
            # R -> R should honor -i n

            # or, when we randomly jump cells, we should more aggreessively return
            # automatically shared variables to sos (done by the following) (#375)
            if kinfo[0] != 'SoS':
                self.switch_kernel('SoS')
                self.switch_kernel(kinfo[0], in_vars, ret_vars)
        elif kinfo[0] == 'SoS':
            # switch from non-sos to sos kernel
            self.handle_magic_put(self.RET_VARS)
            self.RET_VARS = []
            self.kernel = 'SoS'
        elif self.kernel != 'SoS':
            # not to 'sos' (kernel != 'sos'), see if they are the same kernel under
            self.switch_kernel('SoS', in_vars, ret_vars)
            self.switch_kernel(kinfo[0], in_vars, ret_vars)
        else:
            if self._debug_mode:
                self.warn(f'Switch from {self.kernel} to {kinfo[0]}')
            # case when self.kernel == 'sos', kernel != 'sos'
            # to a subkernel
            new_kernel = False
            if kinfo[0] not in self.kernels:
                # start a new kernel
                try:
                    self.kernels[kinfo[0]] = manager.start_new_kernel(
                        startup_timeout=60, kernel_name=kinfo[1], cwd=os.getcwd())
                    new_kernel = True
                except Exception as e:
                    # try toget error message
                    import tempfile
                    with tempfile.TemporaryFile() as ferr:
                        try:
                            # this should fail
                            manager.start_new_kernel(
                                startup_timeout=60, kernel_name=kinfo[1], cwd=os.getcwd(),
                                stdout=subprocess.DEVNULL, stderr=ferr)
                        except:
                            ferr.seek(0)
                            self.warn(f'Failed to start kernel "{kernel}". {e}\nError Message:\n{ferr.read().decode()}')
                    return
            self.KM, self.KC = self.kernels[kinfo[0]]
            self.RET_VARS = [] if ret_vars is None else ret_vars
            self.kernel = kinfo[0]
            if new_kernel and self.kernel in self.supported_languages:
                init_stmts = self.supported_languages[self.kernel](self, kinfo[1]).init_statements
                if init_stmts:
                    self.run_cell(init_stmts, True, False)
            #
            self.handle_magic_get(in_vars)

    def shutdown_kernel(self, kernel, restart=False):
        kernel = self.find_kernel(kernel)[0]
        if kernel == 'SoS':
            # cannot restart myself ...
            self.warn('Cannot restart SoS kernel from within SoS.')
        elif kernel:
            if kernel not in self.kernels:
                self.send_response(self.iopub_socket, 'stream',
                                   dict(name='stdout', text=f'{kernel} is not running'))
            elif restart:
                orig_kernel = self.kernel
                try:
                    # shutdown
                    self.shutdown_kernel(kernel)
                    # switch back to kernel (start a new one)
                    self.switch_kernel(kernel)
                finally:
                    # finally switch to starting kernel
                    self.switch_kernel(orig_kernel)
            else:
                # shutdown
                if self.kernel == kernel:
                    self.switch_kernel('SoS')
                try:
                    self.kernels[kernel][0].shutdown_kernel(restart=False)
                except Exception as e:
                    self.warn(f'Failed to shutdown kernel {kernel}: {e}\n')
                finally:
                    self.kernels.pop(kernel)
        else:
            self.send_response(self.iopub_socket, 'stream',
                               dict(name='stdout', text='Specify one of the kernels to shutdown: SoS{}\n'
                                    .format(''.join(f', {x}' for x in self.kernels))))

    def _parse_error(self, msg):
        self.warn(msg)

    def get_dict_parser(self):
        parser = argparse.ArgumentParser(prog='%dict',
                                         description='Inspect or reset SoS dictionary')
        parser.add_argument('vars', nargs='*')
        parser.add_argument('-k', '--keys', action='store_true',
                            help='Return only keys')
        parser.add_argument('-r', '--reset', action='store_true',
                            help='Rest SoS dictionary (clear all user variables)')
        parser.add_argument('-a', '--all', action='store_true',
                            help='Return all variales, including system functions and variables')
        parser.add_argument('-d', '--del', nargs='+', metavar='VAR', dest='__del__',
                            help='Remove specified variables from SoS dictionary')
        parser.error = self._parse_error
        return parser

    def get_sandbox_parser(self):
        parser = argparse.ArgumentParser(prog='%sandbox',
                                         description='''Execute content of a cell in a temporary directory
                with fresh dictionary (by default).''')
        parser.add_argument('-d', '--dir',
                            help='''Execute workflow in specified directory. The directory
                will be created if does not exist, and will not be removed
                after the completion. ''')
        parser.add_argument('-k', '--keep-dict', action='store_true',
                            help='''Keep current sos dictionary.''')
        parser.add_argument('-e', '--expect-error', action='store_true',
                            help='''If set, expect error from the excution and report
                success if an error occurs.''')
        parser.error = self._parse_error
        return parser

    def handle_magic_dict(self, line):
        'Magic that displays content of the dictionary'
        # do not return __builtins__ beacuse it is too long...
        parser = self.get_dict_parser()
        try:
            args = parser.parse_args(shlex.split(line))
        except SystemExit:
            return

        for x in args.vars:
            if not x in env.sos_dict:
                self.warn('Unrecognized sosdict option or variable name {}'.format(x))
                return

        if args.reset:
            self._reset_dict()
            return

        if args.__del__:
            for x in args.__del__:
                if x in env.sos_dict:
                    env.sos_dict.pop(x)
            return

        if args.keys:
            if args.all:
                self.send_result(env.sos_dict._dict.keys())
            elif args.vars:
                self.send_result(set(args.vars))
            else:
                self.send_result({x for x in env.sos_dict._dict.keys() if not x.startswith('__')} - self.original_keys)
        else:
            if args.all:
                self.send_result(env.sos_dict._dict)
            elif args.vars:
                self.send_result({x: y for x, y in env.sos_dict._dict.items() if x in args.vars})
            else:
                self.send_result({x: y for x, y in env.sos_dict._dict.items() if
                                  x not in self.original_keys and not x.startswith('__')})

    def handle_magic_set(self, options):
        if options.strip():
            # self.send_response(self.iopub_socket, 'stream',
            #    {'name': 'stdout', 'text': 'sos options set to "{}"\n'.format(options)})
            if not options.strip().startswith('-'):
                self.warn(f'Magic %set cannot set positional argument, {options} provided.\n')
            else:
                self.options = options.strip()
                self.send_response(self.iopub_socket, 'stream',
                                   dict(name='stdout', text=f'Set sos options to "{self.options}"\n'))
        else:
            if self.options:
                self.send_response(self.iopub_socket, 'stream',
                                   dict(name='stdout', text=f'Reset sos options from "{self.options}" to ""\n'))
                self.options = ''
            else:
                self.send_response(self.iopub_socket, 'stream',
                                   {'name': 'stdout',
                                    'text': 'Usage: set persistent sos command line options such as "-v 3" (debug output)\n'})

    def handle_magic_get(self, items, from_kernel=None, explicit=False):
        if from_kernel is None or from_kernel.lower() == 'sos':
            # autmatically get all variables with names start with 'sos'
            default_items = [x for x in env.sos_dict.keys() if x.startswith('sos') and x not in self.original_keys]
            items = default_items if not items else items + default_items
            for item in items:
                if item not in env.sos_dict:
                    self.warn(f'Variable {item} does not exist')
                    return
            if not items:
                return
            if self.kernel in self.supported_languages:
                lan = self.supported_languages[self.kernel]
                kinfo = self.find_kernel(self.kernel)
                try:
                    lan(self, kinfo[1]).get_vars(items)
                except Exception as e:
                    self.warn(f'Failed to get variable: {e}\n')
                    return
            elif self.kernel == 'SoS':
                self.warn('Magic %get without option --kernel can only be executed by subkernels')
                return
            else:
                if explicit:
                    self.warn(
                        f'Magic %get failed because the language module for {self.kernel} is not properly installed. Please install it according to language specific instructions on the Running SoS section of the SoS homepage and restart Jupyter server.')
                return
        elif self.kernel.lower() == 'sos':
            # if another kernel is specified and the current kernel is sos
            # we get from subkernel
            try:
                self.switch_kernel(from_kernel)
                self.handle_magic_put(items)
            except Exception as e:
                self.warn(f'Failed to get {", ".join(items)} from {from_kernel}: {e}')
            finally:
                self.switch_kernel('SoS')
        else:
            # if another kernel is specified, we should try to let that kernel pass
            # the variables to this one directly
            try:
                my_kernel = self.kernel
                self.switch_kernel(from_kernel)
                # put stuff to sos or my_kernel directly
                self.handle_magic_put(items, to_kernel=my_kernel, explicit=explicit)
            except Exception as e:
                self.warn(f'Failed to get {", ".join(items)} from {from_kernel}: {e}')
            finally:
                # then switch back
                self.switch_kernel(my_kernel)

    def get_response(self, statement, msg_types, name=None):
        # get response of statement of specific msg types.
        responses = []
        self.KC.execute(statement, silent=False, store_history=False)
        # first thing is wait for any side effects (output, stdin, etc.)
        _execution_state = "busy"
        while _execution_state != 'idle':
            # display intermediate print statements, etc.
            while self.KC.iopub_channel.msg_ready():
                sub_msg = self.KC.iopub_channel.get_msg()
                msg_type = sub_msg['header']['msg_type']
                if self._debug_mode:
                    log_to_file(f'Received {msg_type} {sub_msg["content"]}')
                if msg_type == 'status':
                    _execution_state = sub_msg["content"]["execution_state"]
                else:
                    if msg_type in msg_types and (name is None or sub_msg['content'].get('name', None) in name):
                        if self._debug_mode:
                            log_to_file(f'Capture response: {msg_type}: {sub_msg["content"]}')
                        responses.append([msg_type, sub_msg['content']])
                    else:
                        if self._debug_mode:
                            log_to_file(f'Non-response: {msg_type}: {sub_msg["content"]}')
                        self.send_response(self.iopub_socket, msg_type, sub_msg['content'])
        if not responses and self._debug_mode:
            self.warn(f'Failed to get a response from message type {msg_types}')

        return responses

    def handle_magic_put(self, items, to_kernel=None, explicit=False):
        if self.kernel.lower() == 'sos':
            if to_kernel is None:
                self.warn('Magic %put without option --kernel can only be executed by subkernels')
                return
            # if another kernel is specified and the current kernel is sos
            try:
                # switch to kernel and bring in items
                self.switch_kernel(to_kernel, in_vars=items)
            except Exception as e:
                self.warn(f'Failed to put {", ".join(items)} to {to_kernel}: {e}')
            finally:
                # switch back
                self.switch_kernel('SoS')
        else:
            # put to sos kernel or another kernel
            #
            # items can be None if unspecified
            if not items:
                # we do not simply return because we need to return default variables (with name startswith sos
                items = []
            if self.kernel not in self.supported_languages:
                if explicit:
                    self.warn(f'Subkernel {self.kernel} does not support magic %put.')
                return
            #
            lan = self.supported_languages[self.kernel]
            kinfo = self.find_kernel(self.kernel)
            # pass language name to to_kernel
            if to_kernel:
                objects = lan(self, kinfo[1]).put_vars(items, to_kernel=self.find_kernel(to_kernel)[2])
            else:
                objects = lan(self, kinfo[1]).put_vars(items, to_kernel='SoS')
            if isinstance(objects, dict):
                # returns a SOS dictionary
                try:
                    env.sos_dict.update(objects)
                except Exception as e:
                    self.warn(f'Failed to put {", ".join(items)} to {to_kernel}: {e}')
                    return

                if to_kernel is None:
                    return
                # if another kernel is specified and the current kernel is not sos
                # we need to first put to sos then to another kernel
                try:
                    my_kernel = self.kernel
                    # switch to the destination kernel and bring in vars
                    self.switch_kernel(to_kernel, in_vars=items)
                except Exception as e:
                    self.warn(f'Failed to put {", ".join(items)} to {to_kernel}: {e}')
                finally:
                    # switch back to the original kernel
                    self.switch_kernel(my_kernel)
            elif isinstance(objects, str):
                # an statement that will be executed in the destination kernel
                if to_kernel is None or to_kernel == 'SoS':
                    # evaluate in SoS, this should not happen or rarely happen
                    # because the subkernel should return a dictionary for SoS kernel
                    try:
                        exec(objects, env.sos_dict._dict)
                    except Exception as e:
                        self.warn(f'Failed to put variables {items} to SoS kernel: {e}')
                        return
                try:
                    my_kernel = self.kernel
                    # switch to the destination kernel
                    self.switch_kernel(to_kernel)
                    # execute the statement to pass variables directly to destination kernel
                    self.run_cell(objects, True, False)
                except Exception as e:
                    self.warn(f'Failed to put {", ".join(items)} to {to_kernel}: {e}')
                finally:
                    # switch back to the original kernel
                    self.switch_kernel(my_kernel)
            else:
                self.warn(f'Unrecognized return value of type {object.__class__.__name__} for action %put')
                return

    def handle_magic_pull(self, args):
        from sos.hosts import Host
        if args.config:
            from sos.utils import load_config_files
            load_config_files(args.config)
        cfg = env.sos_dict['CONFIG']
        if args.host == '':
            from sos.hosts import list_queues
            list_queues(cfg, args.verbosity)
            return
        try:
            host = Host(args.host)
            #
            received = host.receive_from_host(args.items)
            #
            msg = '{} item{} received from {}:<br>{}'.format(len(received),
                                                             ' is' if len(received) <= 1 else 's are', args.host,
                                                             '<br>'.join([f'{x} <= {received[x]}' for x in
                                                                          sorted(received.keys())]))
            self.send_response(self.iopub_socket, 'display_data',
                               {
                                   'source': 'SoS',
                                   'metadata': {},
                                   'data': {'text/html': HTML(f'<div class="sos_hint">{msg}</div>').data}
                               })
        except Exception as e:
            self.warn(f'Failed to retrieve {", ".join(args.items)}: {e}')

    def handle_magic_push(self, args):
        from sos.hosts import Host
        if args.config:
            from sos.utils import load_config_files
            load_config_files(args.config)
        cfg = env.sos_dict['CONFIG']
        if args.host == '':
            from .hosts import list_hosts
            list_hosts(cfg, args.verbosity)
            return
        try:
            host = Host(args.host)
            #
            sent = host.send_to_host(args.items)
            #
            msg = '{} item{} sent to {}:<br>{}'.format(len(sent),
                                                       ' is' if len(sent) <= 1 else 's are', args.host,
                                                       '<br>'.join([f'{x} => {sent[x]}' for x in sorted(sent.keys())]))
            self.send_response(self.iopub_socket, 'display_data',
                               {
                                   'source': 'SoS',
                                   'metadata': {},
                                   'data': {'text/html': HTML(f'<div class="sos_hint">{msg}</div>').data}
                               })
        except Exception as e:
            self.warn(f'Failed to send {", ".join(args.items)}: {e}')

    def _interpolate_text(self, text, quiet=False):
        # interpolate command
        try:
            new_text = interpolate(text, local_dict=env.sos_dict._dict)
            if new_text != text and not quiet:
                self.send_response(self.iopub_socket, 'display_data',
                                   {
                                       'source': 'SoS',
                                       'metadata': {},
                                       'data': {
                                           'text/html': HTML(
                                               f'<div class="sos_hint">> {new_text.strip() + "<br>"}</div>').data}
                                   })
            return new_text
        except Exception as e:
            self.warn(f'Failed to interpolate {short_repr(text)}: {e}\n')
            return None

    def handle_magic_preview(self, items, kernel=None, style=None):
        # expand items
        handled = [False for x in items]
        for idx, item in enumerate(items):
            try:
                if os.path.isfile(item):
                    handled[idx] = True
                    self.preview_file(item, style)
                    continue
                else:
                    import glob
                    files = glob.glob(item)
                    if files:
                        for pfile in files:
                            self.preview_file(pfile, style)
                        handled[idx] = True
                        continue
            except Exception as e:
                self.warn(f'\n> Failed to preview file {item}: {e}')
                continue

        # all are files
        if all(handled):
            return

        # non-sos kernel
        use_sos = kernel in ('sos', 'SoS') or (kernel is None and self.kernel == 'SoS')
        orig_kernel = self.kernel
        if kernel is not None and self.kernel != self.find_kernel(kernel)[0]:
            self.switch_kernel(kernel)
        if self._use_panel:
            self.send_frontend_msg('preview-kernel', self.kernel)
        try:
            for item in (x for x, y in zip(items, handled) if not y):
                try:
                    # quoted
                    if (item.startswith('"') and item.endswith('"')) or \
                            (item.startswith("'") and item.endswith("'")):
                        try:
                            item = eval(item)
                        except Exception:
                            pass
                    if use_sos:
                        obj_desc, preview = self.preview_var(item, style)
                        if preview is None:
                            continue
                        else:
                            format_dict, md_dict = preview
                        self.send_frontend_msg('display_data',
                                               {'metadata': {},
                                                'data': {'text/plain': '>>> ' + item + ':\n',
                                                         'text/html': HTML(
                                                             f'<div class="sos_hint">> {item}: {obj_desc}</div>').data
                                                         }
                                                })
                        self.send_frontend_msg('display_data',
                                               {'execution_count': self._execution_count, 'data': format_dict,
                                                'metadata': md_dict})
                    else:
                        self.send_frontend_msg('display_data',
                                               {'metadata': {},
                                                'data': {'text/plain': '>>> ' + item + ':\n',
                                                         'text/html': HTML(
                                                             f'<div class="sos_hint">> {item}:</div>').data
                                                         }
                                                })
                        # evaluate
                        responses = self.get_response(item, ['stream', 'display_data', 'execution_result'])
                        for response in responses:
                            self.send_frontend_msg(response[0], response[1])
                except Exception as e:
                    self.send_frontend_msg('stream',
                                           dict(name='stderr',
                                                text='> Failed to preview file or expression {}{}'.format(
                                                    item, f': {e}' if self._debug_mode else '')))
        finally:
            self.switch_kernel(orig_kernel)

    def handle_magic_cd(self, option):
        if not option:
            return
        to_dir = option.strip()
        try:
            os.chdir(os.path.expanduser(to_dir))
            self.send_response(self.iopub_socket, 'stream',
                               {'name': 'stdout', 'text': os.getcwd()})
        except Exception as e:
            self.warn(f'Failed to change dir to {os.path.expanduser(to_dir)}: {e}')

    def handle_shell_command(self, cmd):
        # interpolate command
        if not cmd:
            return
        from sos.utils import pexpect_run
        try:
            with self.redirect_sos_io():
                pexpect_run(cmd, shell=True,
                            win_width=40 if isinstance(self.cell_idx, int) and self.cell_idx < 0 else 80)
        except Exception as e:
            self.warn(e)

    def run_sos_code(self, code, silent):
        code = dedent(code)
        with self.redirect_sos_io():
            try:
                # record input and output
                fopt = ''
                res = runfile(code=code, raw_args=self.options + fopt, kernel=self)
                self.send_result(res, silent)
            except PendingTasks as e:
                # send cell index and task IDs to frontend
                self.send_frontend_msg('tasks-pending', [self.cell_idx, e.tasks])
                return
            except Exception as e:
                sys.stderr.flush()
                sys.stdout.flush()
                # self.send_response(self.iopub_socket, 'display_data',
                #    {
                #        'source': 'SoS',
                #        'metadata': {},
                #        'data': { 'text/html': HTML('<hr color="black" width="60%">').data}
                #    })
                raise
            except KeyboardInterrupt:
                self.warn('Keyboard Interrupt\n')
                return {'status': 'abort', 'execution_count': self._execution_count}
            finally:
                sys.stderr.flush()
                sys.stdout.flush()
        #
        if not silent and (not hasattr(self, 'preview_output') or self.preview_output):
            # Send standard output
            # if os.path.isfile('.sos/report.md'):
            #    with open('.sos/report.md') as sr:
            #        sos_report = sr.read()
            # with open(self.report_file, 'a') as summary_report:
            #    summary_report.write(sos_report + '\n\n')
            #    if sos_report.strip():
            #        self.send_response(self.iopub_socket, 'display_data',
            #            {
            #                'source': 'SoS',
            #                'metadata': {},
            #                'data': {'text/markdown': sos_report}
            #            })
            #
            if 'input' in env.sos_dict:
                input_files = env.sos_dict['step_input']
                if input_files is None:
                    input_files = []
                else:
                    input_files = [x for x in input_files if isinstance(x, str)]
            else:
                input_files = []
            if 'output' in env.sos_dict:
                output_files = env.sos_dict['step_output']
                if output_files is None:
                    output_files = []
                else:
                    output_files = [x for x in output_files if isinstance(x, str)]
            else:
                output_files = []
            # use a table to list input and/or output file if exist
            if output_files:
                self.send_frontend_msg('preview-input',
                                       f'%preview {" ".join(output_files)}')
                if hasattr(self, 'in_sandbox') and self.in_sandbox:
                    # if in sand box, do not link output to their files because these
                    # files will be removed soon.
                    self.send_frontend_msg('display_data',
                                           {
                                               'source': 'SoS',
                                               'metadata': {},
                                               'data': {'text/html':
                                                            HTML(
                                                                '''<div class="sos_hint"> input: {}<br>output: {}\n</div>'''.format(
                                                                    ', '.join(x for x in input_files),
                                                                    ', '.join(x for x in output_files))).data
                                                        }
                                           })
                else:
                    self.send_frontend_msg('display_data',
                                           {
                                               'source': 'SoS',
                                               'metadata': {},
                                               'data': {'text/html':
                                                            HTML(
                                                                '''<div class="sos_hint"> input: {}<br>output: {}\n</div>'''.format(
                                                                    ', '.join(
                                                                        f'<a target="_blank" href="{x}">{x}</a>' for x
                                                                        in input_files),
                                                                    ', '.join(
                                                                        f'<a target="_blank" href="{x}">{x}</a>' for x
                                                                        in output_files))).data
                                                        }
                                           })
                for filename in output_files:
                    self.preview_file(filename, style=None)

    def preview_var(self, item, style=None):
        if item in env.sos_dict:
            obj = env.sos_dict[item]
        else:
            obj = SoS_eval(item)
        # get the basic information of object
        txt = type(obj).__name__
        # we could potentially check the shape of data frame and matrix
        # but then we will need to import the numpy and pandas libraries
        if hasattr(obj, 'shape') and getattr(obj, 'shape') is not None:
            txt += f' of shape {getattr(obj, "shape")}'
        elif isinstance(obj, Sized):
            txt += f' of length {obj.__len__()}'
        if isinstance(obj, ModuleType):
            return txt, ({'text/plain': pydoc.render_doc(obj, title='SoS Documentation: %s')}, {})
        elif hasattr(obj, 'to_html') and getattr(obj, 'to_html') is not None:
            try:
                from sos.visualize import Visualizer
                result = Visualizer(self, style).preview(obj)
                if isinstance(result, (list, tuple)) and len(result) == 2:
                    return txt, result
                elif isinstance(result, dict):
                    return txt, (result, {})
                elif result is None:
                    return txt, None
                else:
                    raise ValueError(f'Unrecognized return value from visualizer: {short_repr(result)}.')
            except Exception as e:
                self.warn(f'Failed to preview variable: {e}')
                return txt, self.format_obj(obj)
        else:
            return txt, self.format_obj(obj)

    def preview_file(self, filename, style=None):
        if not os.path.isfile(filename):
            self.warn('\n> ' + filename + ' does not exist')
            return
        self.send_frontend_msg('display_data',
                               {'metadata': {},
                                'data': {
                                    'text/plain': f'\n> {filename} ({pretty_size(os.path.getsize(filename))}):',
                                    'text/html': HTML(
                                        f'<div class="sos_hint">> {filename} ({pretty_size(os.path.getsize(filename))}):</div>').data,
                                }
                                })
        previewer_func = None
        # lazy import of previewers
        if self.previewers is None:
            from sos.preview import get_previewers
            self.previewers = get_previewers()
        for x, y, _ in self.previewers:
            if isinstance(x, str):
                if fnmatch.fnmatch(os.path.basename(filename), x):
                    # we load entrypoint only before it is used. This is to avoid
                    # loading previewers that require additional external modules
                    # we can cache the loaded function but there does not seem to be
                    # a strong performance need for this.
                    previewer_func = y.load()
                    break
            else:
                # it should be a function
                try:
                    if x(filename):
                        try:
                            previewer_func = y.load()
                        except Exception as e:
                            self.send_frontend_msg('stream', dict(name='stderr',
                                                                  text=f'Failed to load previewer {y}: {e}'))
                            continue
                        break
                except Exception as e:
                    self.send_frontend_msg('stream', {
                        'name': 'stderr',
                        'text': str(e)})
                    continue
        #
        # if no previewer can be found
        if previewer_func is None:
            return
        try:
            result = previewer_func(filename, self, style)
            if not result:
                return
            if isinstance(result, str):
                self.send_frontend_msg('stream',
                                       {'name': 'stdout', 'text': result})
            elif isinstance(result, dict):
                self.send_frontend_msg('display_data',
                                       {'source': filename, 'data': result, 'metadata': {}})
            elif isinstance(result, [list, tuple]) and len(result) == 2:
                self.send_frontend_msg('display_data',
                                       {'source': filename, 'data': result[0], 'metadata': result[1]})
            else:
                self.send_frontend_msg('stream',
                                       dict(name='stderr', text=f'Unrecognized preview content: {result}'))
        except Exception as e:
            if self._debug_mode:
                self.send_frontend_msg('stream',
                                       dict(name='stderr', text=f'Failed to preview {filename}: {e}'))

    def render_result(self, res):
        if self._render_result is False:
            return res
        if not isinstance(res, str):
            self.warn(
                f'Cannot render result {short_repr(res)} in type {res.__class__.__name__} as {self._render_result}.')
        else:
            # import the object from IPython.display
            mod = __import__('IPython.display')
            if not hasattr(mod.display, self._render_result):
                self.warn(f'Unrecognized render format {self._render_result}')
            else:
                func = getattr(mod.display, self._render_result)
                res = func(res)
        return res

    def send_result(self, res, silent=False):
        # this is Ok, send result back
        if not silent and res is not None:
            format_dict, md_dict = self.format_obj(self.render_result(res))
            self.send_response(self.iopub_socket, 'execute_result',
                               {'execution_count': self._execution_count, 'data': format_dict,
                                'metadata': md_dict})

    def get_kernel_list(self, notebook_kernel_list=None):
        if not hasattr(self, '_kernel_list'):
            from jupyter_client.kernelspec import KernelSpecManager
            km = KernelSpecManager()
            specs = km.find_kernel_specs()
            # get supported languages
            self._kernel_list = []
            lan_map = {}
            for x in self.supported_languages.keys():
                for lname, knames in self.supported_languages[x].supported_kernels.items():
                    for kname in knames:
                        if x != kname:
                            lan_map[kname] = (lname, self.supported_languages[x].background_color,
                                              getattr(self._supported_languages[x], 'options', {}))
            # kernel_list has the following items
            #
            # 1. displayed name
            # 2. kernel name
            # 3. language name
            # 4. color
            for spec in specs.keys():
                if spec == 'sos':
                    # the SoS kernel will be default theme color.
                    self._kernel_list.append(['SoS', 'sos', '', '', {
                        'variable_pattern': r'^[_A-Za-z0-9\.]+\s*$',
                        'assignment_pattern': r'^([_A-Za-z0-9\.]+)\s*=.*$'}])
                elif spec in lan_map:
                    # e.g. ir ==> R
                    self._kernel_list.append([lan_map[spec][0], spec, lan_map[spec][0], lan_map[spec][1],
                                              lan_map[spec][2]])
                else:
                    # undefined language also use default theme color
                    self._kernel_list.append([spec, spec, '', '', {}])
        # now, using a list of kernels sent from the kernel, we might need to adjust
        # our list or create new kernels.
        if notebook_kernel_list is not None:
            for [name, kernel, lan, color] in notebook_kernel_list:
                try:
                    # if we can find the kernel, fine...
                    self.find_kernel(name, kernel, lan, color, notify_frontend=False)
                except Exception as e:
                    # otherwise do not worry about it.
                    env.logger.warning(
                        f'Failed to locate subkernel {name} with kernerl "{kernel}" and language "{lan}": {e}')
        # sort kernel list by name to avoid unnecessary change of .ipynb files
        self._kernel_list.sort(key=lambda x: x[0])
        return self._kernel_list

    def do_execute(self, code, silent, store_history=True, user_expressions=None,
                   allow_stdin=False):
        if self._debug_mode:
            self.warn(code)
        # a flag for if the kernel is hard switched (by %use)
        self.hard_switch_kernel = False
        # evaluate user expression
        try:
            ret = self._do_execute(code=code, silent=silent, store_history=store_history,
                                   user_expressions=user_expressions, allow_stdin=allow_stdin)
        except Exception as e:
            self.warn(e)
            return {'status': 'error',
                    'ename': e.__class__.__name__,
                    'evalue': str(e),
                    'traceback': [],
                    'execution_count': self._execution_count,
                    }

        if ret is None:
            ret = {'status': 'ok',
                   'payload': [], 'user_expressions': {},
                   'execution_count': self._execution_count}

        out = {}
        for key, expr in (user_expressions or {}).items():
            try:
                # value = self.shell._format_user_obj(SoS_eval(expr))
                value = SoS_eval(expr)
                value = self.shell._format_user_obj(value)
            except Exception as e:
                self.warn(f'Failed to evaluate user expression {expr}: {e}')
                value = self.shell._user_obj_error()
            out[key] = value
        ret['user_expressions'] = out
        #
        if not silent and store_history:
            self._real_execution_count += 1
        self._execution_count = self._real_execution_count
        # make sure post_executed is triggered after the completion of all cell content
        self.shell.user_ns.update(env.sos_dict._dict)
        # trigger post processing of object and display matplotlib figures
        self.shell.events.trigger('post_execute')
        # tell the frontend the kernel for the "next" cell
        if store_history:
            self.send_frontend_msg('default-kernel', self.kernel)
        return ret

    def remove_leading_comments(self, code):
        lines = code.splitlines()
        try:
            idx = [x.startswith('#') or not x.strip() for x in lines].index(False)
            return '\n'.join(lines[idx:])
        except Exception:
            # if all line is empty
            return ''

    def _do_execute(self, code, silent, store_history=True, user_expressions=None,
                    allow_stdin=False):
        # if the kernel is SoS, remove comments and newlines
        code = self.remove_leading_comments(code)

        if self.original_keys is None:
            self._reset_dict()
        if code == 'import os\n_pid = os.getpid()':
            # this is a special probing command from vim-ipython. Let us handle it specially
            # so that vim-python can get the pid.
            return
        if self.MAGIC_SKIP.match(code):
            self.warn('The %skip magic is deprecated and will be removed later.')
            return {'status': 'ok', 'payload': [], 'user_expressions': {}, 'execution_count': self._execution_count}
        elif self.MAGIC_RENDER.match(code):
            options, remaining_code = self.get_magic_and_code(code, False)
            parser = self.get_render_parser()
            try:
                args = parser.parse_args(shlex.split(options))
            except SystemExit:
                return
            try:
                self._render_result = args.format
                return self._do_execute(remaining_code, silent, store_history, user_expressions, allow_stdin)
            finally:
                self._render_result = False
        elif self.MAGIC_CAPTURE.match(code):
            options, remaining_code = self.get_magic_and_code(code, False)
            parser = self.get_capture_parser()
            try:
                args = parser.parse_args(shlex.split(options))
            except SystemExit:
                return
            try:
                if not args.__from__:
                    self._capture_result = ''
                return self._do_execute(remaining_code, silent, store_history, user_expressions, allow_stdin)
            finally:
                if args.__to__ and not args.__to__.isidentifier():
                    self.warn(f'Invalid variable name {args.__to__}')
                    self._capture_result = None
                    return
                if args.__append__ and not args.__append__.isidentifier():
                    self.warn(f'Invalid variable name {args.__append__}')
                    self._capture_result = None
                    return
                if self._debug_mode:
                    self.warn(f'Captured {self._capture_result[:40]}')
                if not args.format or args.format == 'text':
                    if args.__from__:
                        # get content from a file
                        try:
                            with open(args.__from__) as ifile:
                                new_var = ifile.read()
                        except Exception as e:
                            self.warn(f'Failed to capture {args.format} from output file {args.__from__}: {e}')
                    else:
                        new_var = self._capture_result
                elif args.format == 'json':
                    import json
                    try:
                        if args.__from__:
                            # get content from a file
                            try:
                                with open(args.__from__) as ifile:
                                    new_var = json.load(ifile)
                            except Exception as e:
                                self.warn(f'Failed to capture output in {args.format} format from output file {args.__from__}: {e}')
                        else:
                            new_var = json.loads(self._capture_result)
                    except Exception as e:
                        self.warn(f'Failed to capture output in JSON format, text returned: {e}')
                        new_var = self._capture_result
                elif args.format == 'csv':
                    try:
                        if args.__from__:
                            # get content from a file
                            try:
                                with open(args.__from__) as ifile:
                                    new_var = pd.read_csv(ifile)
                            except Exception as e:
                                self.warn(f'Failed to capture output in {args.format} format from output file {args.__from__}: {e}')
                        else:
                            with StringIO(self._capture_result) as ifile:
                                new_var = pd.read_csv(ifile)
                    except Exception as e:
                        self.warn(f'Failed to capture output in {args.format} format, text returned: {e}')
                        new_var = self._capture_result
                elif args.format == 'tsv':
                    try:
                        if args.__from__:
                            # get content from a file
                            try:
                                with open(args.__from__) as ifile:
                                    new_var = pd.read_csv(ifile, sep='\t')
                            except Exception as e:
                                self.warn(
                                    f'Failed to capture output in {args.format} format from output file {args.__from__}: {e}')
                        else:
                            with StringIO(self._capture_result) as ifile:
                                new_var = pd.read_csv(ifile, sep='\t')
                    except Exception as e:
                        self.warn(f'Failed to capture output in {args.format} format, text returned: {e}')
                        new_var = self._capture_result
                if args.__to__:
                    env.sos_dict.set(args.__to__, new_var)
                else:
                    if args.__append__ not in env.sos_dict:
                        env.sos_dict.set(args.__append__, new_var)
                    elif isinstance(env.sos_dict[args.__append__], str):
                        if isinstance(new_var, str):
                            env.sos_dict[args.__append__] += new_var
                        else:
                            self.warn(f'Cannot append new content of type {type(new_var).__name__} to {args.__append__} of type {type(env.sos_dict[args.__append__]).__name__}')
                    elif isinstance(env.sos_dict[args.__append__], dict):
                        if isinstance(new_var, dict):
                            env.sos_dict[args.__append__].update(new_var)
                        else:
                            self.warn(f'Cannot append new content of type {type(new_var).__name__} to {args.__append__} of type {type(env.sos_dict[args.__append__]).__name__}')
                    elif isinstance(env.sos_dict[args.__append__], pd.DataFrame):
                        if isinstance(new_var, pd.DataFrame):
                            env.sos_dict.set(args.__append__, env.sos_dict[args.__append__].append(new_var))
                        else:
                            self.warn(f'Cannot append new content of type {type(new_var).__name__} to {args.__append__} of type {type(env.sos_dict[args.__append__]).__name__}')
                    elif isinstance(env.sos_dict[args.__append__], list):
                        env.sos_dict[args.__append__].append(new_var)
                    else:
                        self.warn(f'Cannot append new content of type {type(new_var).__name__} to {args.__append__} of type {type(env.sos_dict[args.__append__]).__name__}')
            self._capture_result = None
        elif self.MAGIC_SESSIONINFO.match(code):
            options, remaining_code = self.get_magic_and_code(code, False)
            parser = self.get_sessioninfo_parser()
            try:
                args = parser.parse_known_args(shlex.split(options))
            except SystemExit:
                return
            self.handle_sessioninfo()
            return self._do_execute(remaining_code, silent, store_history, user_expressions, allow_stdin)
        elif self.MAGIC_TOC.match(code):
            options, remaining_code = self.get_magic_and_code(code, False)
            self.send_frontend_msg('show_toc')
            return self._do_execute(remaining_code, silent, store_history, user_expressions, allow_stdin)
        elif self.MAGIC_DICT.match(code):
            # %dict should be the last magic
            options, remaining_code = self.get_magic_and_code(code, False)
            self.handle_magic_dict(options)
            return self._do_execute(remaining_code, silent, store_history, user_expressions, allow_stdin)
        elif self.MAGIC_CONNECT_INFO.match(code):
            options, remaining_code = self.get_magic_and_code(code, False)
            cfile = find_connection_file()
            with open(cfile) as conn:
                conn_info = conn.read()
            self.send_response(self.iopub_socket, 'stream',
                               {'name': 'stdout', 'text': 'Connection file: {}\n{}'.format(cfile, conn_info)})
            return self._do_execute(remaining_code, silent, store_history, user_expressions, allow_stdin)
        elif self.MAGIC_MATPLOTLIB.match(code):
            options, remaining_code = self.get_magic_and_code(code, False)
            try:
                self.shell.enable_gui = lambda gui: None
                self.shell.enable_matplotlib(options)
            except Exception as e:
                self.warn('Failed to set matplotlib backnd {}: {}'.format(options, e))
            return self._do_execute(remaining_code, silent, store_history, user_expressions, allow_stdin)
        elif self.MAGIC_SET.match(code):
            options, remaining_code = self.get_magic_and_code(code, False)
            self.handle_magic_set(options)
            # self.options will be set to inflence the execution of remaing_code
            return self._do_execute(remaining_code, silent, store_history, user_expressions, allow_stdin)
        elif self.MAGIC_EXPAND.match(code):
            lines = code.splitlines()
            options = lines[0]
            parser = self.get_expand_parser()
            try:
                args = parser.parse_args(options.split()[1:])
            except SystemExit:
                return
            if args.sigil in ('None', None):
                sigil = None
            if args.right_sigil is not None:
                sigil = f'{args.sigil} {args.right_sigil}'
            # now we need to expand the text, but separate the SoS magics first
            lines = lines[1:]
            line_start = 0
            for idx, line in enumerate(lines):
                if line.strip() and not any(line.startswith(f'%{x} ') for x in self.ALL_MAGICS) and not line.startswith('!'):
                    start_line = idx
                    break
            text = '\n'.join(lines[start_line:])
            if sigil is not None and sigil != '{ }':
                from sos.parser import replace_sigil
                text = replace_sigil(text, sigil)
            try:
                interpolated = interpolate(text, local_dict=env.sos_dict._dict)
                remaining_code = '\n'.join(lines[:start_line] + [interpolated]) + '\n'
                # self.options will be set to inflence the execution of remaing_code
                return self._do_execute(remaining_code, silent, store_history, user_expressions, allow_stdin)
            except Exception as e:
                self.warn(e)
                return
        elif self.MAGIC_SHUTDOWN.match(code):
            options, remaining_code = self.get_magic_and_code(code, False)
            parser = self.get_shutdown_parser()
            try:
                args = parser.parse_args(shlex.split(options))
            except SystemExit:
                return
            self.shutdown_kernel(args.kernel if args.kernel else self.kernel, args.restart)
            return self._do_execute(remaining_code, silent, store_history, user_expressions, allow_stdin)
        elif self.MAGIC_CLEAR.match(code):
            options, remaining_code = self.get_magic_and_code(code, False)
            parser = self.get_clear_parser()
            try:
                args = parser.parse_args(options.split())
            except SystemExit:
                return
            # self.cell_idx could be reset by _do_execute
            cell_idx = self.cell_idx
            try:
                return self._do_execute(remaining_code, silent, store_history, user_expressions, allow_stdin)
            finally:
                if args.status:
                    status_style = [self.status_class[x] for x in args.status]
                else:
                    status_style = None
                self.send_frontend_msg('clear-output', [cell_idx, args.all, status_style])
        elif self.MAGIC_FRONTEND.match(code):
            options, remaining_code = self.get_magic_and_code(code, False)
            try:
                parser = self.get_frontend_parser()
                try:
                    args = parser.parse_args(shlex.split(options))
                except SystemExit:
                    return
                self.cell_idx = args.cell_idx
                # 
                #if self.cell_idx is None or self.cell_idx < 0:
                #    self._execution_count = 1
                self._notebook_name = args.filename if args.filename else 'Untitled'
            except Exception as e:
                self.warn('Invalid option "{}": {}\n'.format(options, e))
                return {'status': 'error',
                        'ename': e.__class__.__name__,
                        'evalue': str(e),
                        'traceback': [],
                        'execution_count': self._execution_count,
                        }
            self._use_panel = args.use_panel is True
            if args.list_kernel:
                # https://github.com/jupyter/help/issues/153#issuecomment-289026056
                #
                # when the frontend is refreshed, cached comm would be lost and
                # communication would be discontinued. However, a kernel-list
                # request would be sent by the new-connection so we reset the
                # frontend_comm to re-connect to the frontend.
                self.comm_manager.register_target('sos_comm', self.sos_comm)

            # args.default_kernel should be valid
            if self.find_kernel(args.default_kernel)[0] != self.find_kernel(self.kernel)[0]:
                self.switch_kernel(args.default_kernel)
            #
            if args.cell_kernel == 'undefined':
                args.cell_kernel = args.default_kernel
            #
            original_kernel = self.kernel
            try:
                if self.find_kernel(args.cell_kernel)[0] != self.find_kernel(self.kernel)[0]:
                    self.switch_kernel(args.cell_kernel)
            except Exception as e:
                self.warn(f'Failed to switch to language "{args.cell_kernel}": {e}\n')
                return {'status': 'error',
                        'ename': e.__class__.__name__,
                        'evalue': str(e),
                        'traceback': [],
                        'execution_count': self._execution_count,
                        }
            try:
                if args.resume:
                    self._resume_execution = True
                return self._do_execute(remaining_code, silent, store_history, user_expressions, allow_stdin)
            finally:
                self._resume_execution = False
                if not self.hard_switch_kernel:
                    self.switch_kernel(original_kernel)
        elif self.MAGIC_WITH.match(code):
            options, remaining_code = self.get_magic_and_code(code, False)
            try:
                parser = self.get_with_parser()
                try:
                    args = parser.parse_args(shlex.split(options))
                except SystemExit:
                    return
            except Exception as e:
                self.warn(f'Invalid option "{options}": {e}\n')
                return {'status': 'error',
                        'ename': e.__class__.__name__,
                        'evalue': str(e),
                        'traceback': [],
                        'execution_count': self._execution_count,
                        }
            original_kernel = self.kernel
            if args.restart and args.name in self.kernels:
                self.shutdown_kernel(args.name)
            try:
                self.switch_kernel(args.name, args.in_vars, args.out_vars,
                                   args.kernel, args.language, args.color)
            except Exception as e:
                self.warn(
                    f'Failed to switch to subkernel {args.name} (kernel {args.kernel}, language {args.language}): {e}')
                return {'status': 'error',
                        'ename': e.__class__.__name__,
                        'evalue': str(e),
                        'traceback': [],
                        'execution_count': self._execution_count,
                        }
            try:
                return self._do_execute(remaining_code, silent, store_history, user_expressions, allow_stdin)
            finally:
                self.switch_kernel(original_kernel)
        elif self.MAGIC_USE.match(code):
            options, remaining_code = self.get_magic_and_code(code, False)
            try:
                parser = self.get_use_parser()
                try:
                    args = parser.parse_args(shlex.split(options))
                except SystemExit:
                    return
            except Exception as e:
                self.warn(f'Invalid option "{options}": {e}\n')
                return {'status': 'abort',
                        'ename': e.__class__.__name__,
                        'evalue': str(e),
                        'traceback': [],
                        'execution_count': self._execution_count,
                        }
            if args.restart and args.name in self.kernels:
                self.shutdown_kernel(args.name)
                self.warn(f'{args.name} is shutdown')
            try:
                self.switch_kernel(args.name, args.in_vars, args.out_vars,
                                   args.kernel, args.language, args.color)
                self.hard_switch_kernel = True
                return self._do_execute(remaining_code, silent, store_history, user_expressions, allow_stdin)
            except Exception as e:
                self.warn(
                    f'Failed to switch to subkernel {args.name} (kernel {args.kernel}, language {args.language}): {e}')
                return {'status': 'error',
                        'ename': e.__class__.__name__,
                        'evalue': str(e),
                        'traceback': [],
                        'execution_count': self._execution_count,
                        }
        elif self.MAGIC_GET.match(code):
            options, remaining_code = self.get_magic_and_code(code, False)
            try:
                parser = self.get_get_parser()
                try:
                    args = parser.parse_args(options.split())
                except SystemExit:
                    return
            except Exception as e:
                self.warn(f'Invalid option "{options}": {e}\n')
                return {'status': 'error',
                        'ename': e.__class__.__name__,
                        'evalue': str(e),
                        'traceback': [],
                        'execution_count': self._execution_count,
                        }
            self.handle_magic_get(args.vars, args.__from__, explicit=True)
            return self._do_execute(remaining_code, silent, store_history, user_expressions, allow_stdin)
        elif self.MAGIC_PUT.match(code):
            options, remaining_code = self.get_magic_and_code(code, False)
            try:
                parser = self.get_put_parser()
                try:
                    args = parser.parse_args(options.split())
                except SystemExit:
                    return
            except Exception as e:
                self.warn(f'Invalid option "{options}": {e}\n')
                return {'status': 'error',
                        'ename': e.__class__.__name__,
                        'evalue': str(e),
                        'traceback': [],
                        'execution_count': self._execution_count,
                        }
            self.handle_magic_put(args.vars, args.__to__, explicit=True)
            return self._do_execute(remaining_code, silent, store_history, user_expressions, allow_stdin)
        elif self.MAGIC_PUSH.match(code):
            options, remaining_code = self.get_magic_and_code(code, False)
            try:
                parser = self.get_push_parser()
                try:
                    args = parser.parse_args(options.split())
                except SystemExit:
                    return
            except Exception as e:
                self.warn(f'Invalid option "{options}": {e}\n')
                return {'status': 'error',
                        'ename': e.__class__.__name__,
                        'evalue': str(e),
                        'traceback': [],
                        'execution_count': self._execution_count,
                        }
            self.handle_magic_push(args)
            return self._do_execute(remaining_code, silent, store_history, user_expressions, allow_stdin)
        elif self.MAGIC_PULL.match(code):
            options, remaining_code = self.get_magic_and_code(code, False)
            try:
                parser = self.get_pull_parser()
                try:
                    args = parser.parse_args(options.split())
                except SystemExit:
                    return
            except Exception as e:
                self.warn(f'Invalid option "{options}": {e}\n')
                return {'status': 'error',
                        'ename': e.__class__.__name__,
                        'evalue': str(e),
                        'traceback': [],
                        'execution_count': self._execution_count,
                        }
            self.handle_magic_pull(args)
            return self._do_execute(remaining_code, silent, store_history, user_expressions, allow_stdin)
        elif self.MAGIC_PASTE.match(code):
            options, remaining_code = self.get_magic_and_code(code, True)
            try:
                old_options = self.options
                self.options = options + ' ' + self.options
                try:
                    code = clipboard_get()
                except ClipboardEmpty:
                    raise UsageError("The clipboard appears to be empty")
                except Exception as e:
                    env.logger.warn(f'Failed to get text from the clipboard: {e}')
                    return
                #
                self.send_response(self.iopub_socket, 'stream',
                                   {'name': 'stdout', 'text': code.strip() + '\n## -- End pasted text --\n'})
                return self._do_execute(code, silent, store_history, user_expressions, allow_stdin)
            finally:
                self.options = old_options
        elif self.MAGIC_RUN.match(code):
            # there can be multiple %run magic, but there should not be any other magics
            run_code = code
            run_options = []
            while True:
                if self.MAGIC_RUN.match(run_code):
                    options, run_code = self.get_magic_and_code(run_code, False)
                    run_options.append(options)
                else:
                    break
            # if there are more magics after %run, they will be ignored so a warning
            # is needed.
            if run_code.lstrip().startswith('%'):
                self.warn(f'Magic {run_code.split()[0]} after magic %run will be ignored.')

            # find the global sections of the workflow
            global_sections = ''
            in_global = False
            for line in self._workflow.splitlines():
                if SOS_GLOBAL_SECTION_HEADER.match(line):
                    in_global = True
                elif SOS_SECTION_HEADER.match(line):
                    in_global = False
                if in_global:
                    global_sections += line + '\n'
            if not any(SOS_SECTION_HEADER.match(line) for line in run_code.splitlines()):
                global_sections += '[default]\n'
            # now we need to run the code multiple times with each option
            for options in run_options:
                old_options = self.options
                self.options = options + ' ' + self.options
                try:
                    # %run is executed in its own namespace
                    old_dict = env.sos_dict
                    self._reset_dict()
                    self._workflow_mode = True
                    if self._debug_mode:
                        self.warn(f'Executing\n{global_sections + run_code}')
                    ret = self._do_execute(global_sections + run_code, silent, store_history, user_expressions,
                                           allow_stdin)
                except Exception as e:
                    self.warn(f'Failed to execute workflow: {e}')
                    raise
                finally:
                    old_dict.quick_update(env.sos_dict._dict)
                    env.sos_dict = old_dict
                    self._workflow_mode = False
                    self.options = old_options
            return ret
        elif self.MAGIC_SOSRUN.match(code):
            options, remaining_code = self.get_magic_and_code(code, False)
            old_options = self.options
            self.options = options + ' ' + self.options
            try:
                # %run is executed in its own namespace
                old_dict = env.sos_dict
                self._reset_dict()
                self._workflow_mode = True
                # self.send_frontend_msg('preview-workflow', self._workflow)
                if not self._workflow:
                    self.warn('Nothing to execute (notebook workflow is empty).')
                else:
                    self._do_execute(self._workflow, silent, store_history, user_expressions, allow_stdin)
            except Exception as e:
                self.warn(f'Failed to execute workflow: {e}')
                raise
            finally:
                old_dict.quick_update(env.sos_dict._dict)
                env.sos_dict = old_dict
                self._workflow_mode = False
                self.options = old_options
            return self._do_execute(remaining_code, silent, store_history, user_expressions, allow_stdin)
        elif self.MAGIC_SAVE.match(code):
            if self.kernel != 'SoS':
                # pass the %save magic to underlying kernel
                return self._do_execute(code, silent, store_history, user_expressions, allow_stdin)
            # if sos kernel ...
            options, remaining_code = self.get_magic_and_code(code, False)
            try:
                parser = self.get_save_parser()
                try:
                    args = parser.parse_args(shlex.split(options))
                except SystemExit:
                    return
                filename = os.path.expanduser(args.filename)
                if os.path.isfile(filename) and not args.force:
                    raise ValueError(f'Cannot overwrite existing output file {filename}')

                with open(filename, 'a' if args.append else 'w') as script:
                    script.write('\n'.join(remaining_code.splitlines()).rstrip() + '\n')
                if args.setx:
                    import stat
                    os.chmod(filename, os.stat(filename).st_mode | stat.S_IEXEC)

                self.send_response(self.iopub_socket, 'display_data',
                                   {'source': 'SoS', 'metadata': {},
                                    'data': {
                                        'text/plain': f'Cell content saved to {filename}\n',
                                        'text/html': HTML(
                                            f'<div class="sos_hint">Cell content saved to <a href="{filename}" target="_blank">{filename}</a></div>').data
                                    }
                                    })
            except Exception as e:
                self.warn(f'Failed to save cell: {e}')
                return {'status': 'error',
                        'ename': e.__class__.__name__,
                        'evalue': str(e),
                        'traceback': [],
                        'execution_count': self._execution_count,
                        }
            return self._do_execute(remaining_code, silent, store_history, user_expressions, allow_stdin)
        elif self.MAGIC_SOSSAVE.match(code):
            # get the saved filename
            options, remaining_code = self.get_magic_and_code(code, False)
            try:
                parser = self.get_sossave_parser()
                try:
                    args = parser.parse_args(shlex.split(options))
                except SystemExit:
                    return
                if args.filename:
                    filename = args.filename
                    if filename.lower().endswith('.html'):
                        if args.__to__ is None:
                            ftype = 'html'
                        elif args.__to__ != 'html':
                            self.warn(f'%sossave to an .html file in {args.__to__} format')
                            ftype = args.__to__
                    else:
                        ftype = 'sos'
                else:
                    ftype = args.__to__ if args.__to__ else 'sos'
                    filename = self._notebook_name + '.' + ftype

                filename = os.path.expanduser(filename)

                if os.path.isfile(filename) and not args.force:
                    raise ValueError(f'Cannot overwrite existing output file {filename}')
                # self.send_frontend_msg('preview-workflow', self._workflow)
                if ftype == 'sos':
                    if not args.all:
                        with open(filename, 'w') as script:
                            script.write(self._workflow)
                    else:
                        # convert to sos report
                        from .converter import notebook_to_script
                        arg = argparse.Namespace()
                        arg.all = True
                        notebook_to_script(self._notebook_name + '.ipynb', filename, args=arg, unknown_args=[])
                    if args.setx:
                        import stat
                        os.chmod(filename, os.stat(filename).st_mode | stat.S_IEXEC)
                else:
                    # convert to sos report
                    from .converter import notebook_to_html
                    arg = argparse.Namespace()
                    if args.template == 'default-sos-template':
                        from sos.utils import load_config_files
                        cfg = load_config_files()
                        if 'default-sos-template' in cfg:
                            arg.template = cfg['default-sos-template']
                        else:
                            arg.template = 'sos-report'
                    else:
                        arg.template = args.template
                    notebook_to_html(self._notebook_name + '.ipynb', filename, sargs=arg, unknown_args=[])

                self.send_response(self.iopub_socket, 'display_data',
                                   {'source': 'SoS', 'metadata': {},
                                    'data': {
                                        'text/plain': f'Workflow saved to {filename}\n',
                                        'text/html': HTML(
                                            f'<div class="sos_hint">Workflow saved to <a href="{filename}" target="_blank">{filename}</a></div>').data
                                    }
                                    })
                #
                if args.commit:
                    self.handle_shell_command({'git', 'commit', filename, '-m',
                                               args.message if args.message else f'save {filename}'})
                if args.push:
                    self.handle_shell_command(['git', 'push'])
            except Exception as e:
                self.warn(f'Failed to save workflow: {e}')
                return {'status': 'error',
                        'ename': e.__class__.__name__,
                        'evalue': str(e),
                        'traceback': [],
                        'execution_count': self._execution_count,
                        }
            return self._do_execute(remaining_code, silent, store_history, user_expressions, allow_stdin)
        elif self.MAGIC_RERUN.match(code):
            options, remaining_code = self.get_magic_and_code(code, True)
            old_options = self.options
            self.options = options + ' ' + self.options
            try:
                self._workflow_mode = True
                old_dict = env.sos_dict
                self._reset_dict()
                if not self.last_executed_code:
                    self.warn('No saved script')
                    self.last_executed_code = ''
                return self._do_execute(self.last_executed_code, silent, store_history, user_expressions, allow_stdin)
            except Exception as e:
                self.warn(f'Failed to execute workflow: {e}')
                raise
            finally:
                old_dict.quick_update(env.sos_dict._dict)
                env.sos_dict = old_dict
                self.options = old_options
                self._workflow_mode = False
        elif self.MAGIC_SANDBOX.match(code):
            import tempfile
            import shutil
            options, remaining_code = self.get_magic_and_code(code, False)
            parser = self.get_sandbox_parser()
            try:
                args = parser.parse_args(shlex.split(options))
            except SystemExit:
                return
            self.in_sandbox = True
            try:
                old_dir = os.getcwd()
                if args.dir:
                    args.dir = os.path.expanduser(args.dir)
                    if not os.path.isdir(args.dir):
                        os.makedirs(args.dir)
                    env.exec_dir = os.path.abspath(args.dir)
                    os.chdir(args.dir)
                else:
                    new_dir = tempfile.mkdtemp()
                    env.exec_dir = os.path.abspath(new_dir)
                    os.chdir(new_dir)
                if not args.keep_dict:
                    old_dict = env.sos_dict
                    self._reset_dict()
                ret = self._do_execute(remaining_code, silent, store_history, user_expressions, allow_stdin)
                if args.expect_error and ret['status'] == 'error':
                    # self.warn('\nSandbox execution failed.')
                    return {'status': 'ok',
                            'payload': [], 'user_expressions': {},
                            'execution_count': self._execution_count}
                else:
                    return ret
            finally:
                if not args.keep_dict:
                    env.sos_dict = old_dict
                os.chdir(old_dir)
                if not args.dir:
                    shutil.rmtree(new_dir)
                self.in_sandbox = False
                # env.exec_dir = old_dir
        elif self.MAGIC_PREVIEW.match(code):
            options, remaining_code = self.get_magic_and_code(code, False)
            parser = self.get_preview_parser()
            options = shlex.split(options, posix=False)
            help_option = []
            if ('-s' in options or '--style' in options) and '-h' in options:
                # defer -h to subparser
                options.remove('-h')
                help_option = ['-h']
            try:
                args, style_options = parser.parse_known_args(options)
            except SystemExit:
                return
            #
            style_options.extend(help_option)
            style = {'style': args.style, 'options': style_options}
            #
            if args.off:
                self.preview_output = False
            else:
                self.preview_output = True
            #
            if args.panel:
                self._use_panel = True
            elif args.notebook:
                self._use_panel = False
            # else, use default _use_panel
            try:
                return self._do_execute(remaining_code, silent, store_history, user_expressions, allow_stdin)
            finally:
                # preview workflow
                if args.workflow:
                    import random
                    if self._use_panel:
                        self.send_frontend_msg('preview-input', '%preview --workflow')
                    ta_id = 'preview_wf_{}'.format(random.randint(1, 1000000))
                    self.send_frontend_msg('display_data',
                                       {'metadata': {},
                                        'data':
                                            {'text/plain': self._workflow,
                                             'text/html': HTML(
                                                 f'<textarea id="{ta_id}">{self._workflow}</textarea>').data
                                             }})
                    self.send_frontend_msg('highlight-workflow', ta_id)
                if not args.off and args.items:
                    if args.host is None:
                        if not args.keep_output and self._use_panel:
                            self.send_frontend_msg('preview-input', f'%preview {" ".join(args.items)}')
                        self.handle_magic_preview(args.items, args.kernel, style)
                    elif args.workflow:
                        self.warn('Invalid option --kernel with -r (--host)')
                    elif args.kernel:
                        self.warn('Invalid option --kernel with -r (--host)')
                    else:
                        if args.config:
                            from sos.utils import load_config_files
                            load_config_files(args.config)
                        try:
                            rargs = ['sos', 'preview', '--html'] + options
                            rargs = [x for x in rargs if x not in ('-n', '--notebook', '-p', '--panel')]
                            if self._debug_mode:
                                self.warn(f'Running "{" ".join(rargs)}"')
                            self.send_frontend_msg('preview-input', f'%preview {" ".join(args.items)} -r {args.host}')
                            for msg in eval(subprocess.check_output(rargs)):
                                self.send_frontend_msg(msg[0], msg[1])
                        except Exception as e:
                            self.warn('Failed to preview {} on remote host {}{}'.format(
                                args.items, args.host, f': {e}' if self._debug_mode else ''))
        elif self.MAGIC_CD.match(code):
            options, remaining_code = self.get_magic_and_code(code, False)
            self.handle_magic_cd(options)
            return self._do_execute(remaining_code, silent, store_history, user_expressions, allow_stdin)
        elif self.MAGIC_DEBUG.match(code):
            options, remaining_code = self.get_magic_and_code(code, False)
            parser = self.get_debug_parser()
            try:
                args = parser.parse_args(options.split())
            except SystemExit:
                return
            self._debug_mode = args.status == 'on'
            if self._debug_mode:
                self.warn(remaining_code)
            return self._do_execute(remaining_code, silent, store_history, user_expressions, allow_stdin)
        elif self.MAGIC_TASKINFO.match(code):
            options, remaining_code = self.get_magic_and_code(code, False)
            parser = self.get_taskinfo_parser()
            try:
                args = parser.parse_args(options.split())
            except SystemExit:
                return
            if args.config:
                from sos.utils import load_cfg_files
                load_cfg_files(args.config)
            self.handle_taskinfo(args.task, args.queue)
            return self._do_execute(remaining_code, silent, store_history, user_expressions, allow_stdin)
        elif self.MAGIC_TASKS.match(code):
            options, remaining_code = self.get_magic_and_code(code, False)
            parser = self.get_tasks_parser()
            try:
                args = parser.parse_args(options.split())
            except SystemExit:
                return
            if args.config:
                from sos.utils import load_cfg_files
                load_cfg_files(args.config)
            self.handle_tasks(args.tasks, args.queue if args.queue else 'localhost', args.status, args.age)
            return self._do_execute(remaining_code, silent, store_history, user_expressions, allow_stdin)
        elif code.startswith('!'):
            options, remaining_code = self.get_magic_and_code(code, False)
            self.handle_shell_command(code.split(' ')[0][1:] + ' ' + options)
            return self._do_execute(remaining_code, silent, store_history, user_expressions, allow_stdin)
        elif self.kernel != 'SoS':
            # handle string interpolation before sending to the underlying kernel
            if code:
                self.last_executed_code = code
            # code = self._interpolate_text(code, quiet=False)
            if self.cell_idx is not None:
                self.send_frontend_msg('cell-kernel', [self.cell_idx, self.kernel])
                self.cell_idx = None
            if code is None:
                return
            try:
                return self.run_cell(code, silent, store_history)
            except KeyboardInterrupt:
                self.warn('Keyboard Interrupt\n')
                self.KM.interrupt_kernel()
                return {'status': 'abort', 'execution_count': self._execution_count}
        else:
            if code:
                self.last_executed_code = code
            # run sos
            try:
                self.run_sos_code(code, silent)
                if self.cell_idx is not None:
                    self.send_frontend_msg('cell-kernel', [self.cell_idx, 'SoS'])
                    self.cell_idx = None
                return {'status': 'ok', 'payload': [], 'user_expressions': {}, 'execution_count': self._execution_count}
            except Exception as e:
                self.warn(str(e))
                return {'status': 'error',
                        'ename': e.__class__.__name__,
                        'evalue': str(e),
                        'traceback': [],
                        'execution_count': self._execution_count,
                        }
            finally:
                # even if something goes wrong, we clear output so that the "preview"
                # will not be viewed by a later step.
                env.sos_dict.pop('input', None)
                env.sos_dict.pop('output', None)

    def do_shutdown(self, restart):
        #
        for name, (km, _) in self.kernels.items():
            try:
                km.shutdown_kernel(restart=restart)
            except Exception as e:
                self.warn(f'Failed to shutdown kernel {name}: {e}')

    def __del__(self):
        # upon releasing of sos kernel, kill all subkernels. This I thought would be
        # called by the Jupyter cleanup code or the OS (because subkernels are subprocesses)
        # but they are not.
        self.do_shutdown(False)


if __name__ == '__main__':
    from ipykernel.kernelapp import IPKernelApp

    IPKernelApp.launch_instance(kernel_class=SoS_Kernel)
