"""
Basic classes handling model resolution, system connections and conversion between level of
modelings.
"""
import collections
import importlib
import json
import logging
import os
import warnings
from contextlib import contextmanager
from copy import deepcopy
from enum import Enum
from io import StringIO
from numbers import Number
from pathlib import Path
from typing import (
    Any, Callable, ClassVar, Dict, FrozenSet,
    Iterable, List,
    Optional, Tuple, Union,
)
from types import MappingProxyType

import jsonschema
import numpy
import pandas

from cosapp.core.connectors import Connector, ConnectorError
from cosapp.core.eval_str import EvalString
from cosapp.core.module import Module
from cosapp.core.variableref import VariableReference
from cosapp.core.numerics.basics import MathematicalProblem
from cosapp.core.numerics.boundary import TimeDerivative, TimeUnknown, Unknown
from cosapp.core.numerics.distributions import Distribution
from cosapp.core.numerics.residues import AbstractResidue, Residue
from cosapp.core.time import TimeObserver
from cosapp.ports.enum import CommonPorts, PortType, Scope, Validity
from cosapp.ports.port import ExtensiblePort, Port
from cosapp.ports.variable import RangeValue, Types, Variable
from cosapp.utils.context import ContextLock
from cosapp.utils.helpers import check_arg, is_number, is_numerical
from cosapp.utils.json import JSONEncoder, decode_cosapp_dict
from cosapp.utils.logging import LogFormat, LogLevel, rollover_logfile
from cosapp.utils.pull_variables import pull_variables
from cosapp.utils.find_variables import get_attributes
from cosapp.systems.systemSurrogate import SystemSurrogate
from cosapp.systems.surrogate_models.kriging import FloatKrigingSurrogate

logger = logging.getLogger(__name__)


class ExecutionOrdering(Enum):
    """Enumeration of `System` algorithm to define component execution order."""
    MANUAL = 'manual'


class ConversionType(Enum):
    """Enumeration of `System` conversion type."""

    manual = "manual"
    best_fidelity_to_cost = "best_fidelity_to_cost_ratio"
    high_fidelity = "highest_fidelity"
    low_fidelity = "lowest_fidelity"
    high_cost = "highest_cost"
    low_cost = "lowest_cost"

# Default value use in list_inputs method to be able to tell if the caller set a value to out_stream
_DEFAULT_OUT_STREAM = object()


class System(Module, TimeObserver):
    # TODO check and complete documentation
    """
    A class to describe generic properties and functions of a component that can be single or
    made of child `System`.

    Parameters
    ----------
    name: str
        Name of the `System`

    Attributes
    ----------
    name : str
        `System` name
    inputs : :obj:`collections.OrderedDict` of :obj:`ExtensiblePort`
        Dictionary of `ExtensiblePort` containing the values needed to compute the `System`
    outputs : Dict[ExtensiblePort]
        Dictionary of `ExtensiblePort` containing the values computed by the `System`
    residues : :obj:`collections.OrderedDict` of :obj:float
        Dictionary of residues generated by the `System`
    children : Dict[System]
        Child `System` of this `System`
    parent : System
        Parent `System` of this `System`; None if there is no parent.
    exec_order : List[str]
        Execution order in which the child `System` should be solved.
    properties : Dict[str, Any]

    design_methods : Dict[Equations]
        `System` pre-defined design methods
    name2variable : Dict[str, VariableReference]
        Variable name mapping to its value by reference

    _locked : bool
        if True, `add_input`, `add_output`, `add_inward` and `add_outward` are desactivated. This is
        the default behavior outside the `setup` function.
    _active : bool
        if False, the `System` will not execute its `compute` method
    _is_clean : dict[bool]
        Reflects the status of the inputs and outputs. `clean` status means the group of ports was not updated since
        last computation of the `System`
    _compute_calls: bool
        Store if the `System` was computed at last call or not (due to inhibition of clean status)

    Examples
    --------

    To create your own `System`, you should inherit from this class and define your own `setup` and
    `compute` methods.

    >>> import numpy as np
    >>>
    >>> class RealPort(Port):
    >>>
    >>>     def setup(self):
    >>>         self.add_variable('x',0.)
    >>>
    >>> class InvertedParabola(System):
    >>>
    >>>     def setup(self):
    >>>         self.add_input(RealPort, 'iterative')
    >>>         self.add_input(RealPort, 'target_y')
    >>>         self.add_output(RealPort, 'max_root')
    >>>         self.add_inward({'a': 0.01,
    >>>                        'b': 2.,
    >>>                        'c': 4.})
    >>>         self.add_outward('roots', None)
    >>>         self.add_equation("a * iterative.x**2 + b * iterative.x + c == target_y.x", "y")
    >>>
    >>>     def compute(self):
    >>>
    >>>         discriminant = self.b**2 - 4. * self.a * self.c
    >>>         if discriminant >= 0.:
    >>>             self.roots = ((-self.b + np.sqrt(discriminant)) / (2. * self.a),
    >>>                           (-self.b - np.sqrt(discriminant)) / (2. * self.a))
    >>>             self.max_roots.x = max(self.roots)
    >>>         else:
    >>>             self.roots = None
    >>>             self.max_roots.x = np.nan
    """

    __slots__ = (
        '_context_lock', '_connectors', '_is_clean', '_locked', '_math',
        'design_methods', 'drivers', 'inputs', 'name2variable', 'outputs',
        '__readonly', '_meta','_meta_active_status',
    )

    INWARDS = CommonPorts.INWARDS.value  # type: ClassVar[str]
    OUTWARDS = CommonPorts.OUTWARDS.value  # type: ClassVar[str]

    tags = frozenset()  # type: ClassVar[FrozenSet[str]]
    _user_context = None  # type: ClassVar[Optional[Scope]]

    # Python packages containing components definition
    _components_librairies = ['', ]  # type: ClassVar[List[str]]
    # Is the master system known (i.e. the one from which a simulation has started)
    __master_set: ClassVar[bool] = False

    @classmethod
    @contextmanager
    def set_master(cls, name: str, type_checking: bool=True) -> bool:
        """Set the master System

        Parameters
        ----------
        name : str
            Name of the System calling this context manager
        type_checking : bool, optional
            Whether to activate the type checking in the ports or not (default: True)

        Returns
        -------
        bool
            Is the callee the master System?
        """
        is_master = False
        if not cls.__master_set:
            logger.debug(f"System <{name}> is the execution master.")
            rollover_logfile()  # Create a new logfile
            ExtensiblePort.set_type_checking(type_checking)  # May deactivate type checking
            is_master = True
            cls.__master_set = True

        try:
            yield is_master
        finally:
            if is_master:
                cls.__master_set = False
                ExtensiblePort.set_type_checking(True)  # Reactivate type ckecking
                from cosapp.drivers.optionaldriver import OptionalDriver
                OptionalDriver.set_inhibited(False)

    def __init__(self, name: str, **kwargs):
        """Initialize a System"""
        Module.__init__(self, name)
        TimeObserver.__init__(self, sign_in=False)

        self._math = MathematicalProblem(name, self)  # type: MathematicalProblem
        self.drivers = collections.OrderedDict()  # type: Dict[str, cosapp.Driver]
        self.design_methods = dict()  # type: Dict[str, Tuple[Tuple[Unknown], Tuple[AbstractResidue]]]
        self.__readonly = dict()  # type: Dict[str, Any]

        self._context_lock = ContextLock()  # type: ContextLock
        self.inputs = collections.OrderedDict()  # type: Dict[str, ExtensiblePort]
        self.outputs = dict()  # type: Dict[str, ExtensiblePort]
        # Connectors are grouped in a dictionary where the key is the sink system i.e. the receiving system
        self._connectors = dict()  # type: Dict[str, List[Connector]]
        
        self._locked = False  # type: bool
        self._is_clean = { PortType.IN: False, PortType.OUT: False }  # type: Dict[bool]
        self._meta = None
        self._meta_active_status = False
        # For efficiency purpose, link to object are stored as reference
        # !! Must be the latest attribute set 
        # => KeyError will be raised instead of AttributeError in __setattr__ 
        # => lock __setattr__ on previously defined attributes
        self.name2variable = dict()  # type: Dict[str, VariableReference]

        # Create default ports
        #   for input inwards
        self._add_port(ExtensiblePort(System.INWARDS, PortType.IN))
        #   for internally calculated variables
        self._add_port(ExtensiblePort(System.OUTWARDS, PortType.OUT))

        # Customized subclass `System` before applying user wishes
        kwargs = self._initialize(**kwargs)

        # Customized the `System` according to user wishes
        self.setup(**kwargs)
        self.update()

        self.__enforce_scope()
        self._locked = True

    def _update(self, dt) -> None:
        """Required by TimeObserver base class"""
        pass
        
    def is_clean(self, direction: PortType) -> bool:
        """Are the `System` ports with the given direction cleaned?

        Parameters
        ----------
        direction : PortType
            Direction of interest

        Returns
        -------
        bool
            Clean status
        """
        return self._is_clean[direction]

    def set_clean(self, direction: PortType) -> None:
        """Set to clean ports of a certain direction.

        Parameters
        ----------
        direction : PortType
            Direction to set
        """
        self._is_clean[direction] = True

    def set_dirty(self, direction: PortType) -> None:
        """Set to dirty ports of a certain direction.

        Parameters
        ----------
        direction : PortType
            Direction to set
        """
        self._is_clean[direction] = False
        if (
            direction == PortType.IN
            and self.parent is not None
            and self.parent.is_clean(direction)
        ):
            self.parent.set_dirty(direction)

    def __enforce_scope(self) -> None:
        """Encapsulate input ports for which some variables are out of scope."""
        if self._user_context is None:

            def get_context(
                tags: FrozenSet[str], roles: FrozenSet[FrozenSet[str]]
            ) -> Scope:
                """Get context depending on the matching between user roles and tags.

                If tags is empty => `PRIVATE`
                If one role == all tags => `PRIVATE`

                If one group of one role matches one of the tags => `PROTECTED`

                Else `PUBLIC`

                Parameters
                ----------
                tags : FrozenSet[str]
                roles : FrozenSet[FrozenSet[str]]

                Returns
                -------
                Scope

                Examples
                --------

                >>> tags = frozenset()
                >>> roles = frozenset([frozenset(['Aerodynamics', 'Compressor'])])
                >>> assert get_context(tags, roles) == Scope.PRIVATE
                >>>
                >>> tags = frozenset(['Aerodynamics', 'Compressor'])
                >>> roles = frozenset([frozenset(['Aerodynamics', 'Compressor'])])
                >>> assert get_context(tags, roles) == Scope.PRIVATE
                >>>
                >>> tags = frozenset(['Aerodynamics', 'Compressor'])
                >>> roles = frozenset([frozenset(['Mechanics', 'Compressor'])])
                >>> assert get_context(tags, roles) == Scope.PROTECTED
                >>>
                >>> tags = frozenset(['Aerodynamics', 'Compressor'])
                >>> roles = frozenset([frozenset(['Heat transfert', 'Turbine'])])
                >>> assert get_context(tags, roles) == Scope.PUBLIC

                """
                if len(tags) and tags not in roles:
                    for role in roles:
                        for group in role:
                            if group in tags:
                                return Scope.PROTECTED
                    return Scope.PUBLIC
                else:
                    return Scope.PRIVATE

            # Unsure tags is a frozenset
            self.tags = frozenset(self.tags)

            from cosapp.core.config import CoSAppConfiguration

            user_roles = CoSAppConfiguration().roles

            self._user_context = get_context(self.tags, user_roles)

        if self._user_context != Scope.PRIVATE:  # Some ports may be restrained
            for port in self.inputs.values():
                port.scope_clearance = self._user_context

    def _initialize(self, **kwargs) -> Dict[str, Any]:
        """Hook method to add `System` member before calling `setup` method.

        Parameters
        ----------
        **kwargs : Dict[str, Any]
            Optional keywords arguments of __init__

        Returns
        -------
        Dict[str, Any]
            Optional keywords arguments not consumed by `_initialize`
        """
        return kwargs

    def setup(
        self, **kwargs
    ) -> None:  # TODO update doc for **kwargs and add unit tests
        """`System` port and/or child `System` are defined in this function.

        This function allows to populate a customized `System` class. The helper functions for the
        user are:

        - `add_input` : add an input port
        - `add_output` : add an output port
        - `add_inward` : add a inward variable
        - `add_outward` : add a outward variable
        - `add_child` : add a child `System`

        See Also
        --------
        add_input, add_output, add_inward, add_outward, add_child

        Examples
        --------

        Here is an example of `System` subclassing:

        >>> class AdvancedDuct(System):
        >>>
        >>>     def setup(self):
        >>>         self.add_input(FlowPort, 'in')
        >>>         self.add_output(FlowPort, 'out')
        >>>         self.add_inward({'heat_source': 1.0,
        >>>                          'pressure_loss': 0.01})
        >>>         self.add_outward('wall_temperature', 400.)
        >>>
        >>>         self.add_child(AnotherSystem('system2'))
        """
        pass  # pragma: no cover

    def update(self):
        """Perform complex tasks due to data change.

        Some parameters may be a file used to initialize a complex object. Updating that parameter
        may imply the modification of that complex object. This should be done in this method.

        Examples
        --------

        >>> class TableModel(System):
        >>>
        >>>     def setup(self):
        >>>         # Add a inwards to the file containing the table values
        >>>         self.add_inward('table_file', '{myProject}/ressources/my_table.csv')
        >>>         # Set a object able to interpolate the table
        >>>         self.add_outward('table', Table(self.table_file))
        >>>
        >>>     def update(self):
        >>>         # Update the table object as the source file may have been updated.
        >>>         self.table = Table(self.table_file)

        Notes
        -----

        This method is called systematically after the :py:meth:`~cosapp.systems.system.System.setup` call.
        Otherwise the user is responsible to explicitly call it when needed.
        """
        pass  # pragma: no cover

    def __getattr__(self, name: str) -> Any:
        try:  # Faster than testing `if name in self`
            # Faster to duplicate __getitem__ call than calling it
            variable_ref = self.name2variable[name]
            return variable_ref.value
        except KeyError:
            return super().__getattribute__(name)

    def __setattr__(self, name: str, value: Any) -> None:
        try:  # Faster than testing `if name in self`
            # Faster to duplicate __setitem__ call than calling it
            variable_ref = super().__getattribute__("name2variable")[name]
            variable_ref.value = value
        except KeyError:
            if hasattr(self, name):
                super().__setattr__(name, value)
            else:
                raise AttributeError(
                    f"System {self.__class__.__qualname__!r} has no attribute {name!r}."
                )
        except AttributeError:  
            # Exception catcher to create all initial variables defined prior to name2variable
            # Then KeyError will be raised and it won't be allowed to create new attributes
            super().__setattr__(name, value)

    def __contains__(self, item: str) -> bool:
        return item in self.__readonly or item in self.name2variable

    def __getitem__(self, name: str) -> Any:
        try:
            return getattr(self, name)
        except AttributeError:
            raise KeyError(
                f"Variable {name!r} not found in the context of System {self.name!r}"
            )

    def __setitem__(self, name: str, value: Any) -> None:
        try:
            variable_ref = self.name2variable[name]
        except KeyError:
            if name in self.__readonly:
                raise AttributeError(f"Can't set read-only attribute {self.name}.{name}")
            raise KeyError(
                f"Variable {name!r} not found in the context of System {self.name!r}"
            )
        variable_ref.value = value

    def __repr__(self) -> str:
        return f"{self.name} - {type(self).__name__}"

    def append_name2variable(
        self, additional_mapping: Iterable[Tuple[str, VariableReference]]
    ) -> None:
        """Append the `Iterable` of (`str`, `VariableReference`) `Tuple` to the lookup variables mapping.

        The additional mapping is also transfer upward to the parent `System`.

        Parameters
        ----------
        additional_mapping: Iterable[Tuple[str, VariableReference]]
            Additional list of (str, VariableReference) tuple to append
        """
        # TODO raise error if name already exists (example a inwards variable name == component name)
        additional_mapping = list(additional_mapping)
        self.name2variable.update(additional_mapping)

        if self.parent is not None:
            name = self.name
            rel2absname = lambda item: (f"{name}.{item[0]}", item[1])
            self.parent.append_name2variable(map(rel2absname, iter(additional_mapping)))

    def pop_name2variable(self, keys: Iterable[str]) -> None:
        """Remove the given keys from the name mapping dictionary.

        The keys will be remove from the local mapping. Then the keys list will be sent
        upward to the parent `System` for deletion.

        Parameters
        ----------
        keys: Iterable[str]
            Keys to remove
        """
        keys = list(keys)
        name_mapping = self.name2variable
        for key in keys:
            name_mapping.pop(key)

        if self.parent is not None:
            rel2absname = lambda relname: f"{self.name}.{relname}"
            self.parent.pop_name2variable(map(rel2absname, keys))

    def add_property(self, name: str, value: Any) -> None:
        """Create new read-only property `name`, set to `value`"""
        self.__lock_check("add_property")
        name = Variable.name_check(name)
        self.__check_attr(name, f"cannot add read-only property {name!r};")
        self.__readonly[name] = value
        cls = self.__class__
        def getter(self):
            try:
                return self.__readonly[name]
            except KeyError:
                raise AttributeError(f"{cls.__name__} object {self.name!r} has no attribute {name!r}")
        setattr(cls, name, property(getter))

    @property
    def properties(self) -> Dict[str, Any]:
        """Dict[str, Any]: list of read-only properties and associated values"""
        return MappingProxyType(self.__readonly)

    def add_input(self,
        port_class: "Type[Port]",
        name: str,
        variables: Optional[Dict[str, Any]] = None,
    ) -> ExtensiblePort:
        """Add an input `Port` to the `System`.

        This function cannot be called outside `System.setup`.

        Parameters
        ----------
        port_class: type
            Class of the `Port` to create
        name : str
            `Port` name
        variables : Dict[str, Any], optional
            Dictionary of initial values (default: None)

        Returns
        -------
        ExtensiblePort
            The created port

        Examples
        --------

        >>> class MyModule(System):
        >>>     def setup(self):
        >>>         self.add_input(MyPort, 'input_x')
        >>>         self.add_input(MyPort, 'input_i', {'x': 15.}, all_frozen=True)
        """
        self.__lock_check("add_input")

        # Type validation
        check_arg(port_class, 'port_class', type)
        if not issubclass(port_class, Port):
            raise TypeError(
                f"port_class should be a subclass of Port; got {type(port_class).__name__}."
            )
        check_arg(name, 'name', str)
        check_arg(variables, 'variables', (type(None), dict))

        new_port = port_class(name, PortType.IN, variables=variables)
        self._add_port(new_port)
        return new_port

    def add_output(self,
        port_class: "Type[Port]",
        name: str,
        variables: Optional[Dict[str, Any]] = None
    ) -> ExtensiblePort:
        """Add an output `Port` to the `System`.

        This function cannot be called outside `System.setup`.

        Parameters
        ----------
        port_class: type
            Class of the `Port` to create
        name : str
            `Port` name
        variables : Dict[str, Any], optional
            Dictionary of initial values (default: None)

        Returns
        -------
        ExtensiblePort
            The created port

        Examples
        --------

        >>> class MyModule(System):
        >>>     def setup(self):
        >>>         self.add_output(MyPort, 'output_y')
        >>>         self.add_output(MyPort, 'output_z', {'y': 15.})
        """
        self.__lock_check("add_output")

        # Type validation
        check_arg(port_class, 'port_class', type)
        if not issubclass(port_class, Port):
            raise TypeError(
                f"port_class should be a subclass of Port; got {type(port_class).__name__}."
            )
        check_arg(name, 'name', str)
        check_arg(variables, 'variables', (type(None), dict))

        new_port = port_class(name, PortType.OUT, variables=variables)
        self._add_port(new_port)
        return new_port

    def _add_port(self, port: ExtensiblePort) -> None:
        """Add a port to the system

        Parameters
        ----------
        port : `ExtensiblePort`
            instance of a port class
        """
        if port.name in self:
            raise ValueError(
                "Port {} cannot be added as there already exists an object within system named {}".format(
                    type(port).__qualname__, port.name)
            )

        port.owner = self
        if port.direction == PortType.IN:
            inputs = self.inputs
            if port.name in inputs:
                raise ValueError(f"Port name {port.name!r} already exists as input")
            inputs[port.name] = port
            port_key = (port.name, VariableReference(context=self, mapping=inputs, key=port.name))
        elif port.direction == PortType.OUT:
            outputs = self.outputs
            if port.name in outputs:
                raise ValueError(f"Port name {port.name!r} already exists as output")
            outputs[port.name] = port
            port_key = (port.name, VariableReference(context=self, mapping=outputs, key=port.name))
        else:
            raise ValueError(f"Unknown `PortType` {port.direction}")

        keys = [port_key]
        rel2absname = lambda relname: (
            f"{port.name}.{relname}",
            VariableReference(context=self, mapping=port, key=relname),
        )
        keys.extend(map(rel2absname, iter(port)))
        if isinstance(port, Port):
            attributes = get_attributes(port) - get_attributes(Port) - set(port._variables)
            for attr in attributes:
                keys.append((f"{port.name}.{attr}", VariableReference(context=self, mapping=port, key=attr)))
        self.append_name2variable(keys)

    def __lock_check(self, method: str) -> None:
        """Raises AttributeError if system is locked"""
        if self._locked:
            raise AttributeError(f"`{method}` cannot be called outside `setup`")

    def __check_attr(self, name: str, prefix: str = "") -> None:
        """Raises ValueError if attribute `name` already exists in system"""
        if name in self:
            message = f"{self.name}.{name} already exists"
            prefix = prefix.strip()
            if prefix:
                message = f"{prefix} {message}"
            raise ValueError(message)

    def add_inward(self,
        definition: Union[str, Dict[str, Any]],
        value: Any = 1,
        unit: str = "",
        dtype: Types = None,
        valid_range: RangeValue = None,
        invalid_comment: str = "",
        limits: RangeValue = None,
        out_of_limits_comment: str = "",
        desc: str = "",
        distribution: Optional[Distribution] = None,
        scope: Scope = Scope.PRIVATE,
    ) -> None:
        """Add a inward variable to the `System`.

        A inward variable is calculated by the `System`. But its value is not mandatory in any
        variables fluxes between this `System` and another one.

        An unique inward variable can be defined by providing directly all arguments. And multiple
        inward variables can be defined by passing a `dict` of pair (`str`, `Any`) with an entry for
        each variable.

        This function cannot be called outside :py:meth:`~cosapp.systems.system.System.setup`.

        Parameters
        ----------
        definition : str or Dict[str, Any]
            Name of the unique variable or a dictionary for multiple variables at once
        value : Any, optional
            Value of the variable if `definition` is a `str`; default 1
        unit : str, optional
            Variable unit; default empty string (i.e. dimensionless)
        dtype : type or iterable of types, optional
            Variable type; default None (i.e. type of initial value)
        valid_range : Tuple[Any, Any], optional
            Validity range of the variable; default None (i.e. all values are valid)
        invalid_comment : str, optional
            Comment to show in case the value is not valid; default ''
        limits : Tuple[Any, Any], optional
            Limits over which the use of the model is wrong; default valid_range
        out_of_limits_comment : str, optional
            Comment to show in case the value is not valid; default ''
        desc : str, optional
            Variable description; default ''        
        distribution : Distribution, optional
            Probability distribution of the variable; default None (no distribution)
        scope : Scope {PRIVATE, PROTECTED, PUBLIC}, optional
            Variable visibility; default PRIVATE

        Examples
        --------

        To add one inward variable, arguments must be directly specified.

        >>> system.add_inward('data', 2.)

        To add multiple inward variables, a dictionary with one key per data should be provided.

        >>> system.add_inward({
        >>>     'data1': 42.,
        >>>     'data2': False
        >>> })
        """
        self.__lock_check("add_inward")

        # Type validation
        check_arg(definition, 'definition', (str, dict))

        def add_unique_data(
            name: str,
            value: Any,
            unit: str = "",
            dtype: Types = None,
            valid_range: RangeValue = None,
            invalid_comment: str = "",
            limits: RangeValue = None,
            out_of_limits_comment: str = "",
            distribution: Optional[Distribution] = None,
            desc: str = "",
            scope: Scope = Scope.PRIVATE,
        ) -> None:

            self.__check_attr(name, f"cannot add inward {name!r};")

            inputs = self.inputs

            inputs[System.INWARDS].add_variable(
                name,
                value,
                unit=unit,
                dtype=dtype,
                valid_range=valid_range,
                invalid_comment=invalid_comment,
                limits=limits,
                out_of_limits_comment=out_of_limits_comment,
                desc=desc,
                distribution=distribution,
                scope=scope,
            )

            reference = VariableReference(context=self, mapping=inputs[System.INWARDS], key=name)
            self.append_name2variable(
                [(f"{System.INWARDS}.{name}", reference), (name, reference)]
            )

        if isinstance(definition, dict):
            for key, value in definition.items():
                if isinstance(value, dict):
                    try:
                        add_unique_data(key, **value)
                    except TypeError:
                        add_unique_data(key, value)
                else:
                    add_unique_data(key, value)
        else:
            add_unique_data(
                definition,
                value,
                unit=unit,
                dtype=dtype,
                valid_range=valid_range,
                invalid_comment=invalid_comment,
                limits=limits,
                out_of_limits_comment=out_of_limits_comment,
                desc=desc,
                distribution=distribution,
                scope=scope,
            )

    def add_outward(self,
        definition: Union[str, Dict[str, Any]],
        value: Any = 1,
        unit: str = "",
        dtype: Types = None,
        valid_range: RangeValue = None,
        invalid_comment: str = "",
        limits: RangeValue = None,
        out_of_limits_comment: str = "",
        desc: str = "",
        scope: Scope = Scope.PUBLIC,
    ) -> None:
        """Add a outward variable to the `System`.

        A outward variable is calculated by the `System`. But its value is not mandatory in any
        variables fluxes between this `System` and another one.

        An unique outward variable can be defined by providing directly all arguments. And multiple
        outward variables can be defined by passing a `dict` of pair (`str`, `Any`) with an entry for
        each variable.

        This function cannot be called outside :py:meth:`~cosapp.systems.system.System.setup`.

        Parameters
        ----------
        definition : str or Dict[str, Any]
            Name of the unique variable or a dictionary for multiple variables at once
        value : Any, optional
            Value of the variable if `definition` is a `str`; default 1
        unit : str, optional
            Variable unit; default empty string (i.e. dimensionless)
        dtype : type or iterable of types, optional
            Variable type; default None (i.e. type of initial value)
        valid_range : Tuple[Any, Any], optional
            Validity range of the variable; default None (i.e. all values are valid)
        invalid_comment : str, optional
            Comment to show in case the value is not valid; default ''
        limits : Tuple[Any, Any], optional
            Limits over which the use of the model is wrong; default valid_range
        out_of_limits_comment : str, optional
            Comment to show in case the value is not valid; default ''
        desc : str, optional
            Variable description; default ''
        scope : Scope {PRIVATE, PROTECTED, PUBLIC}, optional
            Variable visibility; default PUBLIC

        Examples
        --------

        To add an unique variable, arguments must be directly specified.

        >>> system.add_outward('info', 2.)

        To add multiple variables, a dictionary with one key per outward variable should be provided.

        >>> system.add_inward({
        >>>     'info1': 42.,
        >>>     'info2': False
        >>> })
        """
        self.__lock_check("add_outward")

        # Type validation
        check_arg(definition, "definition", (str, dict))

        def add_local(
            name: str,
            value: Any,
            unit: str = "",
            dtype: Types = None,
            valid_range: RangeValue = None,
            invalid_comment: str = "",
            limits: RangeValue = None,
            out_of_limits_comment: str = "",
            desc: str = "",
            scope: Scope = Scope.PUBLIC,
        ) -> None:
            self.__check_attr(name, f"cannot add outward {name!r}")

            outputs = self.outputs
            outputs[System.OUTWARDS].add_variable(
                name,
                value,
                unit=unit,
                dtype=dtype,
                valid_range=valid_range,
                invalid_comment=invalid_comment,
                limits=limits,
                out_of_limits_comment=out_of_limits_comment,
                desc=desc,
                scope=scope,
            )

            reference = VariableReference(context=self, mapping=outputs[System.OUTWARDS], key=name)
            self.append_name2variable(
                [(f"{System.OUTWARDS}.{name}", reference), (name, reference)]
            )

        if isinstance(definition, dict):
            for key, value in definition.items():
                if isinstance(value, dict):
                    try:
                        add_local(key, **value)
                    except TypeError:
                        add_local(key, value)
                else:
                    add_local(key, value)
        else:
            add_local(
                definition,
                value,
                unit=unit,
                dtype=dtype,
                valid_range=valid_range,
                invalid_comment=invalid_comment,
                limits=limits,
                out_of_limits_comment=out_of_limits_comment,
                desc=desc,
                scope=scope,
            )

    def add_transient(self,
            name: str,
            der: str,  # time derivative of `name`
            desc: str = None,
            max_time_step: Union[Number, str] = numpy.inf,
            max_abs_step: Union[Number, str] = numpy.inf,
        ) -> None:
        """
        Declare a transient variable, defined implicitly by the expression of its time derivative, and add
        a time-dependent unknown to the mathematical problem.
        Transients can be used withinn the system, but their time evolution is computed outside the system, by a time driver.

        If `name` already exists, it must refer to an inward.
        If `name` does not exist, a new inward `name` is created on the fly.

        This function cannot be called outside :py:meth:`~cosapp.systems.system.System.setup`.

        Parameters
        ----------
        name : str
            Name of the transient variable
        der : str
            Expression of the time derivative, used for integration during time driver execution.
            The type and unit of the transient variable are deduced from `der`.
        desc : str, optional
            Variable description; default None        
        max_time_step : Number or evaluable expression (str), optional
            Maximum time step admissible for the integration of the variable (for stability, typically).
        max_abs_step : Number or evaluable expression compatible with transient variable, optional
            Maximum variable step admissible over one time step (for stability, typically).

        Examples
        --------

        >>> system.add_inward('v', numpy.zeros(3), desc='Velocity')
        >>> system.add_inward('a', numpy.zeros(3), desc='Acceleration')
        >>> system.add_inward('area', 1.0)
        >>> system.add_outward('flowrate', desc='Volumetric flowrate')
        >>> system.add_transient('x', der='v')  # x defined by dx/dt = v
        >>> system.add_transient('v', der = 'a',
                max_time_step = '0.1 / max(norm(a), 1e-9)',
            )
        >>> system.add_transient('h',
                der = 'flowrate / area',
                max_abs_step = 0.1,
            )
        """
        check_arg(name, "name", str)
        check_arg(der, "der", str)

        _, value, dtype = TimeUnknown.der_type(der, self)

        if name in self:
            _, container, key = self.name2variable[name]
            ok = isinstance(container, ExtensiblePort) and (container.direction == PortType.IN)
            if not ok:
                raise TypeError(
                    "Only input variables can be declared as transient variables; got {!r} in {!r}".format(
                        name, self.name)
                )
            # Check that existing inward is compatible with derivative expression
            declared_value = container[key]
            if isinstance(value, numpy.ndarray):
                ok = (numpy.shape(declared_value) == value.shape)
            else:
                ok = is_number(declared_value)
            if not ok:
                raise TypeError(
                    "Type incompatibility: {} {!r} cannot be the time derivative of {} variable {!r}".format(
                        dtype.__name__, der, type(declared_value).__name__, name))

        else:  # Create new inward on the fly
            logger.info(f"Creation of new time-dependent inward {name!r} within system {self.name!r}")

            if desc is None:
                str_der = self.str_der(1)
                desc = f"Transient variable defined as {str_der(name)} = {der}"

            # Need to copy the value to avoid vector linked by reference between variable and its derivative
            #  See https://gitlab.safrantech.safran/cosapp/cosapp/issues/179
            self.add_inward(name, value=deepcopy(value), desc=desc, dtype=dtype)

        self._math.add_transient(name, der, max_time_step, max_abs_step)

    def add_rate(self,
            name: str,
            source: Any,  # `name` is the time derivative of `source`
            initial_value: Union[Number, numpy.ndarray, str] = None,
            desc: str = None,
        ) -> None:
        """
        Add a variable monitoring the rate-of-change of a given quantity (referred to as `source`) as the system evolves.
        Rates are defined by their source, and are updated by the time driver computing the time evolution of the system.

        This function cannot be called outside :py:meth:`~cosapp.systems.system.System.setup`.

        Parameters
        ----------
        name : str
            Name of the transient variable
        source : Any
            Expression of the quantity of interest whose rate will be computed.
            The type and unit of the rate are deduced from `source`.
        desc : str, optional
            Variable description; default None        
        initial_value : Number/array, or evaluable expression (str), optional.
            Initial value of the rate (otherwise unknown, as rates are only updated from the first time step on).
            If specified, it must be consistent with the type of source (scalar vs. array).

        Examples
        --------

        >>> system.add_inward('U', 0.1, desc='Input voltage')
        >>> system.add_rate('dUdt', source='U')  # dUdt defined as dU/dt
        """
        check_arg(name, "name", str)
        self.__check_attr(name, f"cannot add rate {name!r};")

        _, value, dtype = TimeDerivative.source_type(source, self)

        if isinstance(value, numpy.ndarray):
            cast = lambda x: numpy.array(x)
        else:
            cast = lambda x: x

        def numerical_value(expression):
            if expression is None:
                return None
            if isinstance(expression, str):
                value = EvalString(expression, self).eval()
            elif isinstance(expression, EvalString):
                value = expression.eval()
            else:
                value = expression
            return cast(value)

        if initial_value is not None and isinstance(value, numpy.ndarray):
            # If given, `initial_value` must conform to `value.shape`
            num_val = numerical_value(initial_value)
            check_arg(numpy.asarray(num_val), 'initial_value', numpy.ndarray, lambda array: array.shape == value.shape)

        if name in self:
            _, container, key = self.name2variable[name]
            ok = isinstance(container, ExtensiblePort) and (container.direction == PortType.IN)
            if not ok:
                raise TypeError(
                    f"Only input variables can be declared as rates; got {self.name}.{name}"
                )
            # Check that existing inward is compatible with requested rate
            declared_value = container[key]
            if isinstance(value, numpy.ndarray):
                ok = (numpy.shape(declared_value) == value.shape)
            else:
                ok = is_number(declared_value)
            if not ok:
                raise TypeError(
                    "Type incompatibility: {} {!r} cannot be the time derivative of {} {}".format(
                        type(declared_value).__name__, name, dtype.__name__, source))
            self[name] = numerical_value(initial_value)

        else:  # Create new inward on the fly
            logger.info(f"Creation of new time-dependent inward {name!r} within system {self.name!r}")

            if desc is None:
                str_der = self.str_der(1)
                desc = str_der(source)

            value = numerical_value(initial_value)
            self.add_inward(name, value=value, desc=desc, dtype=dtype)

        self._math.add_rate(name, source, initial_value)

    @staticmethod
    def str_der(order: int) -> Callable[[str,], str]:
        """Derivate name factory.
        
        Parameters
        ----------
        order : int
            Derivative order
        
        Returns
        -------
        Callable[[int, ], str]
            Function generating the derivate name from the variable name
        """
        if order == 1:
            return lambda s: f"d({s})/dt"
        elif order > 1:
            return lambda s: "d^{0}({1})/dt^{0}".format(s, order)
        return lambda s: s

    def _precompute(self) -> None:
        if len(self._math.rates) > 0:
            self.set_dirty(PortType.IN)  # ensured that system is recomputed at first time step

    def check(self, name: Optional[str] = None) -> Union[Dict[str, Validity], Validity]:
        """Get variable value validity for a given variable, port or system or all of them.

        If `name` is not provided, returns a dictionary with the validity of all `System` variables;
        i.e. all variables in input and output ports including inwards and outwards of this system and
        all its children.
        Else only, the validity for the given variable will be returned.

        Parameters
        ----------
        name : str, optional
            Variable name

        Returns
        -------
        Dict[str, Validity] or Validity
            (Dictionary of) the variable(s) value validity
        """
        # TODO unit tests
        if name is None:
            validity = dict()

            def get_variables(object_name: str) -> bool:
                if isinstance(self[object_name], (System, ExtensiblePort)):
                    return False
                elif "." in object_name:
                    owner_name, _ = object_name.rsplit(".", maxsplit=1)
                    owner = self[owner_name]
                    return isinstance(
                        owner, ExtensiblePort
                    )  # Only valid owner is a port
                else:  # Shortcut to inwards or outwards => not taken
                    return False

            for obj_name in filter(get_variables, self.name2variable):
                validity[obj_name] = self.check(obj_name)
            return validity

        elif name in self:
            obj = self[name]
            if isinstance(obj, (System or ExtensiblePort)):
                return obj.check()
            else:
                reference = self.name2variable[name]
                return reference.mapping.check(reference.key)

        else:
            raise AttributeError(f"Variable {name} does not exist in System {self.name}")

    def is_running(self) -> bool:
        """Is this System in execution?
        
        Returns
        -------
        bool
            In execution status
        """
        return self._context_lock.is_locked()

    @property
    def residues(self) -> Dict[str, Residue]:
        """Dict[str, Residue] : Get the residues for the current `System`."""
        # MappingProxyType forbids external modification
        return MappingProxyType(self._math.residues)

    @property
    def unknowns(self) -> Dict[str, Unknown]:
        """Dict[str, Unknown] : Get the unknowns for the current `System`."""
        # MappingProxyType forbids external modification
        return MappingProxyType(self._math.unknowns)

    @property
    def transients(self) -> Dict[str, TimeUnknown]:
        """Returns a dictionary containing all transient unknowns in current system tree"""
        # MappingProxyType forbids external modification
        return MappingProxyType(self._math.transients)

    @property
    def rates(self) -> Dict[str, TimeDerivative]:
        """Returns a dictionary containing all time derivatives (rates) in current system tree"""
        # MappingProxyType forbids external modification
        return MappingProxyType(self._math.rates)

    @property
    def connectors(self) -> Dict[str, Connector]:
        """MappingProxyType[str, Connector] : Connectors contained by the system."""
        # MappingProxyType forbids external modification
        out = dict()
        for connectors in self._connectors.values():
            for connector in connectors:
                out[connector.name] = connector
        return MappingProxyType(out)

    def get_unsolved_problem(self) -> MathematicalProblem:
        """Returns the unsolved mathematical problem.

        Returns
        -------
        MathematicalProblem
            The unsolved mathematical problem
        """
        new = MathematicalProblem('off-design', self)

        for name in ('residues', 'unknowns', 'transients', 'rates'):
            var_dict = getattr(new, name)
            var_dict.update(getattr(self._math, name))

        def transfer_unknown(unknown, name):
            options = { attr: getattr(unknown, attr)
                for attr in ('max_abs_step', 'max_rel_step', 'lower_bound', 'upper_bound') }
            new.add_unknown(name, **options)

        def transfer_transient(unknown, name):
            ref = unknown.context.name2variable[unknown.name]
            new.add_transient(
                name, 
                der=unknown.der,
                max_time_step=unknown.max_time_step_expr,
                pulled_from=unknown.pulled_from or ref
            )

        def transfer_rate(unknown, name):
            new.add_rate(name, source=unknown.source_expr,
                initial_value=unknown.initial_value_expr)

        for child in self.children.values():
            if child.is_standalone():
                continue

            child_problem = child.get_unsolved_problem()
            connectors = self._connectors.get(child.name, [])
            transfer_items = [
                (child_problem.unknowns, transfer_unknown),
                (child_problem.transients, transfer_transient),
                (child_problem.rates, transfer_rate),
            ]

            for connector in connectors:
                # Transfer unknowns, transients and rates to parent (when necessary)
                for unknowns, transfer in transfer_items:
                    for name in list(unknowns.keys()):
                        unknown = unknowns[name]
                        port = unknown.context[unknown.port]
                        if (port.owner.name in self.children
                            and port is connector.sink 
                            and unknown.variable in connector.variable_mapping):
                            # Port is connected => remove unknown
                            unknowns.pop(name)
                            if connector.source.owner is self:
                                # Transfer unknown to parent level
                                src = connector.variable_mapping[unknown.variable]
                                transfer(unknown, f"{connector.source.name}.{src}")

            new.extend(child_problem, copy=False)

        return new

    def add_child(self,
        child: "System",
        execution_index: Optional[int] = None,
        pulling: Optional[Union[str, List[str], Dict[str, str]]] = None,
    ) -> "System":
        """Add a child `System` to the current `System`.

        When adding a child `System`, it is possible to specified its position in the execution
        order.

        Child `Port`, `inwards` and `outwards` can also be pulled at the parent level by providing either
        the name of the port/inward/outward or a list of them or the name mapping of the child element
        (dictionary keys) to the parent element (dictionary values).
        If the argument is not a dictionary, the name in the parent system will be the same as in
        the child.

        Parameters
        ----------
        child: System
            `System` to add to the current `System`
        execution_index: int, optional
            Index of the execution order list at which the `System` should be inserted;
            default latest.
        pulling: str or List[str] or Dict[str, str], optional
            Map of child ports to pulled ports at the parent system level; default None (no pulling)
        """
        # Type validation
        check_arg(child, 'child', System)
        check_arg(execution_index, 'execution_index', (type(None), int), lambda i: i is None or i >= 0)
        check_arg(pulling, 'pulling', (type(None), str, list, dict, set))

        child = super().add_child(child, execution_index)

        if child.name in self.name2variable:
            if isinstance(self[child.name], System):
                logger.warning(
                    f"A subsystem named {child.name} already exists within system {self.name}."
                    f" Child system {child!r} will overwrite it."
                )
            else:
                raise ValueError(f"An object named {child.name!r} already exists in system {self}")

        keys = [(child.name, VariableReference(context=self, mapping=self.children, key=child.name))]
        # Append child system name mapping to current mapping
        rel2absname = lambda item: (f"{child.name}.{item[0]}", item[1])
        mapping_generator = map(rel2absname, child.name2variable.items())
        keys.extend(mapping_generator)
        self.append_name2variable(keys)

        if pulling is not None:
            try:
                pull_variables(child, pulling)
            except ConnectorError as error:
                self.pop_child(child.name)
                raise ConnectorError(error.message)

        return child

    def pop_child(self, name: str) -> Optional["System"]:
        """Remove the `System` called `name` from the current top `System`.

        Parameters
        ----------
        name: str
            Name of the `System` to remove

        Returns
        -------
        `System` or None
            The removed `System` or None if no match found
        """
        child = super().pop_child(name)
        
        # Remove the connection to the system
        if name in self._connectors:
            self._connectors.pop(name)

        for c in self._connectors:
            self._connectors[c] = list(filter(lambda c: c.source.owner is not child, self._connectors[c]))

        # Remove the System from the name mapping
        keys = [name]
        rel2absname = lambda relname: f"{name}.{relname}"
        mapping_generator = map(rel2absname, iter(child.name2variable))
        keys.extend(mapping_generator)
        self.pop_name2variable(keys)

        return child

    def add_driver(self, driver: "cosapp.drivers.Driver") -> "cosapp.drivers.Driver":
        """Add a driver to this system.

        Parameters
        ----------
        driver : Driver
            Driver to add

        Returns
        -------
        Driver
            The added driver
        """
        from cosapp.drivers import Driver

        check_arg(driver, 'driver', Driver)

        name = driver.name
        default_name = f"default_driver_for_{hex(id(self))}"

        if default_name in self.drivers and name != default_name:
            self.drivers.clear()

        if name in self.drivers:
            logger.warning(f"Driver {name!r} already exists and will be replaced")

        driver.owner = self
        driver.parent = None  # Clear driver parent in case of driver reused
        self.drivers[name] = driver

        return self.drivers[name]

    def add_unknown(self,
            name: Union[str, Iterable[Union[dict, str]]],
            max_abs_step: Number = numpy.inf,
            max_rel_step: Number = numpy.inf,
            lower_bound: Number = -numpy.inf,
            upper_bound: Number = numpy.inf
    ) -> "MathematicalProblem":
        """Add unknown variables.

        You can set variable one by one or provide a list of dictionary to set multiple variable at once. The
        dictionary key are the arguments of this method.

        Parameters
        ----------
        name : str or Iterable of dictionary or str
            Name of the variable or list of variable to add
        max_rel_step : float, optional
            Maximal relative step by which the variable can be modified by the numerical solver; default numpy.inf
        max_abs_step : float, optional
            Maximal absolute step by which the variable can be modified by the numerical solver; default numpy.inf
        lower_bound : float, optional
            Lower bound on which the solver solution is saturated; default -numpy.inf
        upper_bound : float, optional
            Upper bound on which the solver solution is saturated; default numpy.inf

        Returns
        -------
        MathematicalProblem
            The modified MathematicalSystem
        """

        def create_unknown(
                name: str,
                max_abs_step: Number = numpy.inf,
                max_rel_step: Number = numpy.inf,
                lower_bound: Number = -numpy.inf,
                upper_bound: Number = numpy.inf
        ):
            unknown = Unknown(self, name, max_abs_step, max_rel_step, lower_bound, upper_bound)
            # Remove existing unknown if user wants to update the parameters.
            if unknown.name in self._math.unknowns:
                self._math.unknowns.pop(unknown.name)
            self._math.add_unknown([unknown])

        if isinstance(name, str):
            create_unknown(name, max_abs_step, max_rel_step, lower_bound, upper_bound)
        else:
            for item in name:
                create_unknown(item, max_abs_step, max_rel_step, lower_bound, upper_bound)

        return self._math

    def add_equation(self,
            equation: Union[str, Iterable[Union[dict, str, Tuple[str, str]]]],
            name: Optional[str] = None,
            reference: Union[Number, numpy.ndarray, str] = 1) -> "MathematicalProblem":
        """Add residue equation.

        You can add residue equation one by one or provide a list of dictionary to add multiple equation at once. The
        dictionary key are the arguments of this method.

        Parameters
        ----------
        equation : str or Iterable of str of the kind 'lhs == rhs'
            Equation or list of equations to add
        name : str, optional
            Name of the equation; default None => 'lhs == rhs'
        reference : Number, numpy.ndarray or "norm", optional
            Reference value(s) used to normalize the equation; default is 1.
            If value is "norm", actual reference value is estimated from order of magnitude.

        Returns
        -------
        MathematicalProblem
            The modified MathematicalSystem
        """
        self.__lock_check("add_equation")

        return self._math.add_equation(equation, name, reference)

    def add_design_method(self, name: str) -> MathematicalProblem:
        """Add a design method to the `System`

        A design method is a set of free variables and equations that defines a way to design the `System`
        It is a easy way to pre-define a design of this `System` for users

        The returned mathematical system is empty. It should be populated with the needed variables and
        equations (see Examples).

        Parameters
        ----------
        name: str
            The name of the design method

        Returns
        -------
        MathematicalSystem
            The mathematical system to solve to fulfill the method.

        Examples
        --------

        >>> system1.add_design_method("method1") \
        >>>        .add_unknown([
        >>>             dict(name="x", max_rel_step=0.1),
        >>>             "y"
        >>>         ]) \
        >>>        .add_equation([
        >>>             "u == 0",
        >>>             "v == 800"
        >>>         ])
        """
        self.__lock_check("add_design_method")
        self.design_methods[name] = mathpb = MathematicalProblem(name, self)
        return mathpb

    def design(self, method: str) -> MathematicalProblem:
        """Returns the chosen group of design equations.

        Parameters
        ----------
        method : str
            Name of the group of equations to extract

        Returns
        -------
        MathematicalProblem
            Mathematical system to solve for the chosen method
        """
        return self.design_methods[method]

    def log_debug_message(
        self,
        handler: "HandlerWithContextFilters",
        record: logging.LogRecord,
        format: LogFormat = LogFormat.RAW
    ) -> bool:
        """Callback method on the system to log more detailed information.
        
        This method will be called by the log handler when :py:meth:`~cosapp.utils.logging.LoggerContext.log_context`
        is active if the logging level is lower or equals to VERBOSE_LEVEL. It allows
        the object to send additional log message to help debugging a simulation.

        .. note::
            logger.log method cannot be used here. Use handler.handle(record)

        Parameters
        ----------
        handler : HandlerWithContextFilters
            Log handler on which additional message should be published.
        record : logging.LogRecord
            Log record
        format : LogFormat
            Format of the message

        Returns
        -------
        bool
            Should the provided record be logged?
        """
        def print_port(header, ports) -> List[str]:
            msg = [f"#### {header}", ]

            for name, port in ports.items():
                if len(port) == 0:
                    continue
                msg.append(f"- {name}:")

                for var in port:
                    value = port[var]
                    if is_number(value):
                        v = f"{value:.5g}"
                    else:
                        v = f"{value!s}"
                    msg.append(f"  - {var} = {v}")
            return "\n".join(msg)
        message = record.getMessage()
        activate = getattr(record, "activate", None)
        emit_record = super().log_debug_message(handler, record, format)

        if message.endswith("call_setup_run") or message.endswith("call_clean_run"):
            emit_record =  False

        elif activate == True:
            handler.log(
                LogLevel.FULL_DEBUG,
                print_port(f"{self!s} - Inputs", self.inputs),
                name=logger.name
            )
            emit_record = False

        elif activate == False:
            handler.log(
                LogLevel.FULL_DEBUG,
                print_port(f"{self!s} - Outputs", self.outputs),
                name=logger.name
            )
            emit_record = False

        return emit_record

    def call_setup_run(self, skip_driver: bool = False) -> None:
        """Execute `setup_run` recursively on all modules.
        
        Parameters
        ----------
        skip_drivers : bool
            Skip calling :py:meth:`cosapp.drivers.driver.Driver.setup_run`
        """
        if not skip_driver:
            for driver in self.drivers.values():
                driver.call_setup_run()
        super().call_setup_run()

    def call_clean_run(self, skip_driver: bool = False) -> None:
        """Execute `clean_run` recursively on all modules.
        
        Parameters
        ----------
        skip_drivers : bool
            Skip calling :py:meth:`cosapp.drivers.driver.Driver.clean_run`
        """
        super().call_clean_run()
        if not skip_driver:
            for driver in self.drivers.values():
                driver.call_clean_run()

        self.set_dirty(PortType.IN)
        self.set_dirty(PortType.OUT)

    def _postcompute(self) -> None:
        """Actions performed after the `System.compute` call."""
        if self.is_clean(PortType.IN):
            self.set_clean(PortType.OUT)
        else:
            self.set_clean(PortType.IN)
            self.set_dirty(PortType.OUT)

            # Evaluate the residues
            for r in self.residues.values():
                r.update()

    def run_once(self) -> None:
        """Run the system once.

        Execute the model of this `System` and its children in the execution order.

        Notes
        -----
        The drivers are not executed when calling this method; only the physical model.
        """
        with System.set_master(repr(self)) as is_master:
            if is_master:
                logger.debug("Start setup_run recursive calls.")
                self.call_setup_run(skip_driver=True)

            with self.log_context(" - run_once"):
                if self.is_active():
                    self._precompute()

                    if not self.is_clean(PortType.IN):
                        logger.debug(f"Call {self.name}.compute_before")
                        with self._context_lock:
                            self.compute_before()
                    else:
                        logger.debug(f"Skip {self.name}.compute_before - Clean inputs")

                    if not (self.is_clean(PortType.IN) and self.is_clean(PortType.OUT)):
                        for child in self.exec_order:
                            # Update connectors
                            for connector in self._connectors.get(child, []):
                                connector.transfer()
                            # Execute the child
                            logger.debug(f"Call {self.name}.{child}.run_once()")
                            self.children[child].run_once()

                        # Pull values from inside Modules to top shelve border
                        for connector in self._connectors.get(self.name, []):
                            connector.transfer()
                    else:
                        logger.debug(f"Skip {self.name} children execution - Clean interfaces")

                    if not self.is_clean(PortType.IN):
                        logger.debug(f"Call {self.name}.compute")
                        self._compute_calls += 1
                        with self._context_lock:
                            if self.active_surrogate:
                                self._meta.compute()
                            else :
                                self.compute()
                    else:
                        logger.debug(f"Skip {self.name}.compute - Clean inputs")

                    self._postcompute()
                    self.computed.emit()
                else:
                    logger.debug(f"Skip {self.name} execution - Inactive")

            if is_master:
                logger.debug("Start clean_run recursive calls.")
                self.call_clean_run(skip_driver=True)

    @property
    def any_active_driver(self) -> bool:
        return any(driver.is_active() for driver in self.drivers.values())

    def run_drivers(self) -> None:
        """Run the drivers defined on this `System`.
        """
        with System.set_master(repr(self), type_checking=False) as is_master:
            if is_master:
                self.open_loops()
                logger.debug("Start setup_run recursive calls.")
                self.call_setup_run()

            if self.drivers and self.any_active_driver:
                if self.is_standalone():  # System not standalone can't set the mathematical problem
                    self._set_execution_order()
                    logger.debug(f"Exec order for {self.name}: {self.exec_order}")

                for driver in self.drivers.values():
                    logger.debug(f"Call driver {driver.name}.run_once on {self.name}")
                    driver.run_once()
            else:
                self.run_children_drivers()

            if is_master:
                logger.debug("Start clean_run recursive calls.")
                self.call_clean_run()
                self.close_loops()

    def run_children_drivers(self) -> None:
        """Solve the children `System` in the execution order.
        """
        with System.set_master(repr(self), type_checking=False) as is_master:
            if is_master:
                logger.debug("Start setup_run recursive calls.")
                self.call_setup_run()

            with self.log_context(" - run_children_drivers"):
                if self.is_active():
                    self._precompute()

                    if not self.is_clean(PortType.IN):
                        logger.debug(f"Call {self.name}.compute_before")
                        with self._context_lock:
                            self.compute_before()
                    else:
                        logger.debug(f"Skip {self.name}.compute_before - Clean inputs")

                    if not (self.is_clean(PortType.IN) and self.is_clean(PortType.OUT)):
                        for child in self.exec_order:
                            # Update connectors
                            for connector in self._connectors.get(child, []):
                                connector.transfer()
                            # Execute the child
                            self.children[child].run_drivers()

                        # Pull values from inside Modules to top shelve border
                        for connector in self._connectors.get(self.name, []):
                            connector.transfer()
                    else:
                        logger.debug(f"Skip {self.name} children execution - Clean interfaces")

                    if not self.is_clean(PortType.IN):
                        self._compute_calls += 1
                        with self._context_lock:
                            if self.active_surrogate:
                                self._meta.compute()
                                logger.debug(f"Call {self.name}.meta.compute")
                            else :
                                self.compute()
                                logger.debug(f"Call {self.name}.compute")
                    else:
                        logger.debug(f"Skip {self.name}.compute - Clean inputs")

                    self._postcompute()
                    self.computed.emit()
                else:
                    logger.debug(f"Skip {self.name} execution - Inactive")

            if is_master:
                logger.debug("Start clean_run recursive calls.")
                self.call_clean_run()

    def is_standalone(self) -> bool:  # TODO name is confusing as the real answer is I have a non-linear solver
        """Is this System able to solve itself?

        Returns
        -------
        bool
            Ability to solve the system or not.
        """
        for driver in self.drivers.values():
            if driver.is_standalone():
                return True

        return False

    def connect(self,
        port1: ExtensiblePort,
        port2: ExtensiblePort,
        mapping: Union[str, List[str], Dict[str, str], None] = None,
    ) -> None:
        """Connect two ports together. 

        This method connect ``port1`` to ``port2``. If no mapping is provided, connection
        will be made between variables based on their names. If a name mapping is provided as a list, 
        the name should be present in both port. And if the mapping is specified as a dictionary, 
        the keys belong to ``port1`` and the values to ``port2``.

        An connection is oriented (i.e. the direction of value transfer is fixed). The decision tree
        is as following:

        - If one port is of type input and the other one of type output, the connection flows from
          the output to the input.
        - Else if one port (portA) belong to the parent `System` of the other port (portB), the
          connection flows from portA to portB if portA is an input. And it flows from portB to
          portA if portA is an output.
        - Else if both are inputs, the port is pulled on the parent and connected to the two children.

        Connectors can only connect systems at the same level or a system with its parent. In all
        cases the connectors will be stored by the parent hosting the two connected systems.

        Parameters
        ----------
        port1 : ExtensiblePort
            The port to connect.
        port2 : ExtensiblePort
            The port to connect to.
        mapping : str or List[str] or Dict[str, str], optional
            (List of) common name(s) or mapping name dictionary; default None (i.e. no mapping).
        
        Examples
        --------

        Here is an example, in which we assume a port ``p1`` has inputs of a system ``s1``
        that come from some output port ``p2`` of another system ``s2``. The container system
        being called ``top``:

        >>> class DemoPort(Port):
        >>>     def setup(self):
        >>>         self.add_variable('a',1)
        >>>         self.add_variable('b',2)
        >>>         self.add_variable('c',3)
        >>>
        >>> class DummySystem(Port):
        >>>     def setup(self):
        >>>         self.add_input(DemoPort, 'p1')
        >>>         self.add_output(DemoPort, 'p2')
        >>>
        >>> top = System('top')
        >>> top.add(DummySystem('s1'))
        >>> top.add(DummySystem('s2'))
        >>>
        >>> top.connect(top.s1.p1, top.s2.p2, 'a')
        >>> top.s2.p2.a = 4
        >>> top.run_once()
        >>> assert top.s1.p1.a == top.s2.p2.a
        >>>
        >>> top.connect(top.s1.p1, top.s2.p2, {'b': 'c'})
        >>> top.run_once()
        >>> assert top.s1.p1.b == top.s2.p2.c
        """

        # Type validation
        check_arg(port1, 'port1', ExtensiblePort)
        check_arg(port2, 'port2', ExtensiblePort)
        check_arg(mapping, 'mapping', (str, list, dict, type(None)))

        # Generate mapping dictionary
        if mapping is None:
            mapping = dict([(v, v) for v in port1 if v in port2])
        elif isinstance(mapping, str):
            mapping = dict([(mapping, mapping)])
        elif not isinstance(mapping, dict):
            mapping = dict(zip(mapping, mapping))

        def create_connector(sink, source, mapping):
            # Additional validation for sink Port
            # - Source should be Port (the opposite case is possible == connect sensor)
            # - If mapping does not cover all variables, print a log message

            source_name = source.contextual_name
            sink_name = sink.contextual_name

            if isinstance(sink, Port):
                if not (len(sink) == len(source) == len(mapping)):
                    absent = [f"{sink.name}.{v}" for v in sink if v not in mapping]
                    absent.extend([f"{source.name}.{v}" for v in source if v not in list(mapping.values())])
                    logger.debug(
                        "Partial connection between {!r} and {!r}. "
                        "Variables ({}) are not part of the mapping.".format(
                            sink_name, source_name, ", ".join(absent)
                        )
                    )
                    
            name = f"{source_name}_to_{sink_name}".replace(".", "_")

            new_connector = Connector(name, sink, source, mapping)
            this_connectors = self.connectors
            # Check that the variable is set only once
            for connector in this_connectors.values():
                if connector.sink is sink:
                    for variable in connector.variable_mapping:
                        if variable in new_connector.variable_mapping:
                            raise ConnectorError(
                                f"{sink_name}.{variable} is already set by {connector}"
                            )

            if new_connector.name in this_connectors:  # Sink and source are already connected
                the_connector = this_connectors[new_connector.name]
                connector_mapping = the_connector.variable_mapping

                for new_target, new_source in new_connector.variable_mapping.items():
                    connector_mapping[new_target] = new_source
                    the_connector.update_unit_conversion(new_target)
            else:
                target = new_connector.sink.owner.name
                if target not in self._connectors:
                    self._connectors[target] = list()
                self._connectors[target].append(new_connector)

        port1_dir = port1.direction
        port2_dir = port2.direction

        if not (port2.owner is None or port1.owner is None):
            msg = "Ports {!r} and {!r} cannot be connected".format(
                port1.contextual_name, port2.contextual_name)
            if port1.owner is port2.owner:
                raise ConnectorError(
                    f"{msg}. Connecting ports of the same system is forbidden."
                )
            if port1_dir != port2_dir:  # one port is IN and the other is OUT
                if port1.owner is port2.owner.parent or port2.owner is port1.owner.parent:
                    raise ConnectorError(
                        f"{msg}, as they are of different types, and link a child to its parent."
                    )
                elif not (port1.owner.parent is port2.owner.parent is self):
                    raise ConnectorError(
                        f"{msg}. Only ports belonging to direct children of {self.name!r} can be connected."
                    )
            else:
                if not (
                    (self is port1.owner.parent is port2.owner) 
                    or (self is port1.owner is port2.owner.parent)
                    or (port1.owner in self.children.values() and port2.owner in self.children.values())
                ):
                    raise ConnectorError(
                        f"{msg}. Same-direction ports can only be connected between a child and its parent, "
                        "or between same-group systems."
                    )

        if port1_dir == PortType.IN and port2_dir == PortType.OUT:
            create_connector(port1, port2, mapping)
        elif port1_dir == PortType.OUT and port2_dir == PortType.IN:
            create_connector(port2, port1, dict((v, k) for k, v in mapping.items()))
        else:
            if not (port2.owner is None or port1.owner is None):
                if port1.owner is port2.owner.parent:
                    if port1_dir == PortType.IN:
                        create_connector(
                            port2, port1, dict((v, k) for k, v in mapping.items())
                        )
                    else:  # port1_dir == PortType.OUT
                        create_connector(port1, port2, mapping)
                    return
                elif port2.owner is port1.owner.parent:
                    if port2_dir == PortType.IN:
                        create_connector(port1, port2, mapping)
                    else:  # port2_dir == PortType.OUT
                        create_connector(
                            port2, port1, dict((v, k) for k, v in mapping.items())
                        )
                    return

            if port1_dir == PortType.IN:  # Both port are inputs
                # In this case the port is transfer on the parent to serve as
                # true source for all subsystems.
                # For Port, the port is duplicated in the parent. For ExtensiblePort,
                # the variable is added in the INWARDS port of the parent.

                for p1_var, p2_var in mapping.items():
                    connected = False
                    # Check if the variable is connected
                    for c1 in filter(lambda c: c.sink is port1, self._connectors.get(port1.owner.name, [])):
                        if p1_var in c1.variable_mapping:  # Already connected
                            # Check that the other variable is not connected
                            for c2 in filter(lambda c: c.sink is port2, self._connectors.get(port2.owner.name, [])):
                                if p2_var in c2.variable_mapping.values():
                                    raise ConnectorError("{}.{} is already connected to {}.{}".format(
                                            port1.contextual_name, p1_var, 
                                            port2.contextual_name, p2_var,
                                        )
                                    )

                            # Connect the other variable
                            self.connect(port2, c1.source, {p2_var: c1.variable_mapping[p1_var]})
                            connected = True
                            break

                    if connected:
                        continue

                    # Check if the other variable is connected
                    for c2 in filter(lambda c: c.sink is port2, self._connectors.get(port2.owner.name, [])):
                        if p2_var in c2.variable_mapping:
                            # We know that p1_var is not connected
                            # Connect the variable
                            self.connect(port1, c2.source, {p1_var: c2.variable_mapping[p2_var]})
                            connected = True
                            break

                    if connected:
                        continue

                    # General case
                    # create the pulled port
                    if port2.name == System.INWARDS:
                        pulled_port_name = System.INWARDS
                        source_name = f"{port2.owner.name}_{p2_var}"
                        pull_variables(port2.owner, {p2_var: source_name})
                    else:
                        pulled_port_name = f"{port2.owner.name}_{port2.name}"
                        source_name = p2_var
                        # Is the full port pulled or only a portion of it?
                        if all([v in mapping.values() for v in port2]):
                            pull_variables(port2.owner, {port2.name: pulled_port_name})
                        else:
                            pulled_port = port2.copy(pulled_port_name)
                            self._add_port(pulled_port)
                            self.connect(port2, pulled_port, p1_var)
                        
                    # create the other connection
                    self.connect(port1, self.inputs[pulled_port_name], {p1_var: source_name})

            else:
                raise ConnectorError(
                    "Port {} cannot be connected to port {} as they are both outputs.".format(
                        port1.contextual_name, port2.contextual_name))

    def open_loops(self):
        """Open closed loops in children relations."""
        logger.debug(f"Call {self.name}.open_loops")
        connections_to_open = list()  # type: List[Connector]

        # Add top system in the execution loop so connection to it won't be opened
        execution_loop = [self.name, ]  # type: List[str]
        for name in self.exec_order:
            child = self.children[name]
            execution_loop.append(name)

            child.open_loops()

            # Check that child inputs do not depend on not already executed system
            connectors = list()
            for connector in self._connectors.get(name, []):
                if connector.source.owner.name in execution_loop:
                    connectors.append(connector)
                else:
                    connections_to_open.append(connector)
            if name in self._connectors:  # Set only if the standard system has connections
                self._connectors[name] = connectors

        for connector in connections_to_open:
            logger.debug(f"Connector {connector!r} to be opened.")
            # Compute the connection residues 
            # => pull source values only if max(residues) > 1e-4
            #   History: converged non-linear system was iterating again if re-run twice in raw
            connection = IterativeConnector(connector)
            max_res = numpy.max(
                numpy.abs(
                    connection.get_unsolved_problem().residues_vector
                )
            )

            if max_res > 1e-4:
                connector.source.owner.set_dirty(PortType.OUT)
                connector.transfer()  # Pull source values for a better initialization.

            self.add_child(connection)
            self.connect(
                connection.inputs[IterativeConnector.GUESS], 
                connector.sink, 
                list(connector.variable_mapping)
            )
            self.connect(
                connection.inputs[IterativeConnector.RESULT], 
                connector.source, 
                list(connector.variable_mapping.values())
            )

    def close_loops(self):
        """Close loops opened to allow system resolution."""
        logger.debug(f"Call {self.name}.close_loops")

        def restore_connector(connection: IterativeConnector) -> None:
            # Remove the IterativeConnector instance
            self.pop_child(connection.name)

            sink, source, mapping = connection.get_connection()

            # We need to remove the pulled port or variables.
            if sink.name == System.INWARDS:
                _src_port = self.inputs[System.INWARDS]
                src_names = [f"{sink.owner.name}_{v}" for v in mapping]
            else:
                _src_port = self.inputs[f"{sink.owner.name}_{sink.name}"]
                src_names = None

            keys = []
            rel2absname = lambda relname: f"{_src_port.name}.{relname}"
            keys.extend(map(rel2absname, src_names or mapping))
            if sink.name == System.INWARDS:
                keys.extend(src_names)
                for n in src_names:
                    self.inputs[System.INWARDS].remove_variable(n)
            else:
                keys.append(_src_port.name)
                self.inputs.pop(_src_port.name)
            self.pop_name2variable(keys)

            # Remove connections
            for idx, c in enumerate(self._connectors[sink.owner.name][:]):
                if c.source is _src_port:
                    if src_names is not None:
                        c.remove_variables(
                            # remove_variables needs the sink name not the source
                            [k for k, v in c.variable_mapping.items() if v in src_names]
                        )
                    
                    if len(c.variable_mapping) == 0 or src_names is None:
                        self._connectors[sink.owner.name].pop(idx)

            # Restore the connections
            self.connect(sink, source, mapping)

        for child in list(self.children.values()):
            if isinstance(child, IterativeConnector):
                restore_connector(child)
            else:
                child.close_loops()

    def convert_to(self, *args, **kwargs) -> None:
        """Convert system into another `System`.
        Note: not implemented for class `System`.

        Raises
        ------
        TypeError
            If the current `System` cannot be converted.
        """
        # TODO: this method should be removed altogether
        raise TypeError(
            f"Cannot convert {self.name!r}: system is not part of a family"
        )

    @classmethod
    def check_config_dict(cls, params: dict) -> None:
        """Check if the provided dictionary respects the convention of a model file.

        Parameters
        ----------
        params : dict
            Dictionary to be tested

        Raises
        ------
        jsonschema.exceptions.ValidationError
            If the provided dictionary does not conform to the JSON schema.
        """
        # Check that the file input file respects the JSON schema
        path = os.path.dirname(os.path.abspath(__file__))
        with open(os.path.join(path, 'system.schema.json')) as fp:
            config_schema = json.load(fp)
        jsonschema.validate(params, config_schema)

    @classmethod
    def load(cls, filepath: Union[str, Path, StringIO]) -> 'System':
        """Load configuration from file-like object.

        Parameters
        ----------
        filepath : str or Path or file-like
            Filepath or file-like object (i.e. has a .read() method)

        Returns
        -------
        System
            The built system.
        """
        if isinstance(filepath, (str, Path)):
            with open(filepath) as f:
                params = json.load(f)
        elif hasattr(filepath, 'read'):
            params = json.load(filepath)
        else:
            raise TypeError("Input parameter should be a filepath or file-like object.")

        cls.check_config_dict(params)
        # Remove schema reference entry
        params.pop('$schema', None)
        # Convert variables if needed
        params = decode_cosapp_dict(params)

        return cls.load_from_dict(*params.popitem())

    @classmethod
    def load_from_dict(cls, name: str, parameters: dict) -> 'System':
        """Instantiate a `System` from its name and its parameters.

        The construction of the `System` is done recursively from the lower level. Therefore this
        function is also called recursively returning to the upper level, the lower `System` and
        its connections to the upper `System` via a dictionary. If no connections are to forwarded
        the second returned argument is `None`.

        Parameters
        ----------
        name : str
            Identifier of the system
        parameters : dict
            The dictionary containing the system definition

        Returns
        -------
        System
            The generated system
        """
        # TODO The all process of saving and reading from a file should be overhaul to take into account missing attr
        #  - setup kwargs
        #  - design methods (?)
        # Moreover what should be the source of truth JSON or Python? Currently reading from JSON duplicates Python
        # work as first the object is instantiated. Then its children are removed to create them (and the associated
        # connections from the JSON file.

        def get_system_class(system_type):
            check_arg(system_type, 'Class name', str)
            cls_name = system_type.rsplit('.', maxsplit=1)
            system_class = None
            found_module = ''
            component_module = ''
            if len(cls_name) == 2:
                component_module, cls_name = cls_name
            elif len(cls_name) == 1:
                cls_name = cls_name[0]

            for lib in System._components_librairies:
                try:
                    if len(component_module) > 0:
                        mod_name = f"{lib}.{component_module}"
                        if mod_name.startswith('.'):
                            mod_name = mod_name[1:]
                        lib_modules = importlib.import_module(mod_name)
                    elif len(lib) > 0:
                        lib_modules = importlib.import_module(lib)
                    else:
                        raise ImportError
                except ImportError:
                    # Try the next possible Python package to find the module
                    continue
                else:
                    # Found an existing module
                    found_module = lib_modules.__file__
                    try:
                        system_class = getattr(lib_modules, cls_name)
                    except AttributeError:
                        # Class not in module
                        continue
                    else:
                        break

            if system_class is None:
                if len(found_module) == 0:
                    message = "System {} is not in one of the listed component librairies:\n{}".format(
                        cls_name, System._components_librairies)
                else:
                    message = f"Class {cls_name} was not found in module: \n{found_module}"
                raise AttributeError(message)
            elif not issubclass(system_class, System):
                raise AttributeError(
                    f"Class '{system_class.__name__}' does not inherit from base class 'System'")

            return system_class

        system_type = parameters.get('class', 'System')

        if system_type == 'System':
            system_class = System
        else:
            system_class = get_system_class(system_type)

        top_system = system_class(name=name)
        for name in top_system.children:
            top_system.name2variable.pop(name)
        # Remove children and connectors --> the source of truth is the json file
        top_system.children.clear()
        top_system._connectors.clear()

        top_system.__readonly = parameters.get('properties', {}).copy()

        for name, value in parameters.get('inputs', {}).items():
            top_system[name] = value

        for name_system, params in parameters.get('subsystems', {}).items():
            subsystem = cls.load_from_dict(name_system, params)
            top_system.add_child(subsystem)
            
        for connection in parameters.get('connections', []):
            if len(connection) == 2:
                sink, source = connection
                mapping = None
            else:
                sink, source, mapping = connection
            top_system.connect(top_system[sink], top_system[source], mapping)

        if 'exec_order' in parameters:
            top_system.exec_order = list()
            for c in parameters['exec_order']:
                if c in top_system.children:
                    top_system.exec_order.append(c)
                else:
                    logger.warning(
                        "Component {} is specified in the execution order of {}, but is not defined as subsystem".format(
                            c, top_system.name))

            if len(top_system.exec_order) != len(top_system.children):
                logger.warning(
                    "Execution order length does not match the number of subsystems in system {}".format(
                        top_system.name))

        return top_system

    def save(self, fp, indent: int = 2, sort_keys: bool = True) -> None:
        """Serialize the `System` as a JSON formatted stream to fp.

        Parameters
        ----------
        fp : str or Path or file-like
            A .write()-supporting file-like object or the filename
        indent : int, optional
            Indentation in the file (default: 2)
        sort_keys : bool, optional
            Sort the keys in alphabetic order (default: False)
        """
        if isinstance(fp, (str, Path)):
            with open(fp, 'w') as filepath:
                filepath.write(self.to_json(indent, sort_keys))
        else:
            fp.write(self.to_json(indent, sort_keys))


    def export_structure(self) -> Dict:
        """
        Export current system structure to a dictionary, which contains
        the definition of all ports, the structure of sub-system and the 
        connections between ports.
        """
        
        port_cls_data = {}
        sys_data = self.__to_dict(True, port_cls_data)
        return {"Ports":port_cls_data, "Systems": sys_data }

    def to_dict(self) -> Dict:
        """
        Public API to export system to a dictionary
        """
        return self.__to_dict(False)

    def __to_dict(self, with_struct: bool, port_cls_data: Dict = None) -> Dict:
        """Convert the `System` to a dictionary.
        In default mode, `with_struct` flag is `False`, only export input ports
        and its variable values.

        In other case where `with_struct` flag is `True`, export all ports with
        values and class name. `port_cls_data` can be provided to recover the
        definitions of all ports inside system.
        
        
        Parameters
        ----------
        with_struct : bool, optional
            Flag to export output ports and the class name of ports (default: False).

        port_cls_data : Dict, optional
            A dictionary to hold the definition of all ports inside system.

        Returns
        -------
        dict
            A dictionary defining fully the `System`
        """
        # TODO Fred - BUG Information missing from dictionary conversion
        # - kwargs of setup - e.g. Shaft system of librairies_sae has its ports functions of kwargs
        # - design equations - are not saved (should they be?)
        tmp = dict()

        if self.__class__.__qualname__ == 'System':
            tmp['class'] = 'System'  # Trick to allow container `System` without special type
        else:
            tmp['class'] = f"{self.__module__}.{self.__class__.__qualname__}"

        properties = self.properties
        if len(properties) > 0:
            tmp['properties'] = properties

        if with_struct:            
            for direction in ["inputs", "outputs"]:
                temp_dict = {}
                for name, port in getattr(self,direction ).items():
                    if name not in ["inwards", "outwards"]:
                        port_cls_name = port.__class__.__qualname__
                        if port_cls_data is not None and isinstance(port_cls_data, Dict) and port_cls_name not in port_cls_data :
                            port_cls_data[port_cls_name] = {}
                            for var_name, var_data in port._variables.items():
                                var_dict =  var_data.to_dict()
                                port_cls_data[port_cls_name][var_name] = var_dict

                    port_dict = port.to_dict(with_struct)
                    temp_dict.update(port_dict)

                if len(temp_dict) > 0:
                    tmp[direction] = temp_dict           
        else:
            inputs = {}
            for name, port in self.inputs.items():
                port_dict = port.to_dict()
                inputs.update(port_dict)
            if len(inputs) > 0:
                tmp['inputs'] = inputs

        connections = []
        for connectors in self._connectors.values():
            for c in connectors:
                connections.append(c.to_dict()[c.name])
        if len(connections) > 0:
            tmp['connections'] = connections

        if len(self.children) > 0:
            tmp['subsystems'] = dict()
            for name, component in self.children.items():
                tmp['subsystems'][name] = component._System__to_dict(with_struct, port_cls_data)[name]

            if self.exec_order:
                tmp['exec_order'] = list(self.exec_order)

        return {self.name: tmp}

    def to_json(self, indent: int = 2, sort_keys: bool = True) -> str:
        """Return a string in JSON format representing the `System`.

        Parameters
        ----------
        indent : int, optional
            Indentation of the JSON string (default: 2)
        sort_keys : bool, optional
            Sort the keys in alphabetic order (default: False)

        Returns
        -------
        str
            String in JSON format
        """
        dict_repr = self.to_dict()
        # If this is updated => workspace template should be updated too.
        dict_repr['$schema'] = "0-3-0/system.schema.json"  # Add self referencing to system JSON version - provision
        return json.dumps(dict_repr, indent=indent, sort_keys=sort_keys, cls=JSONEncoder)

    def to_d3(self, show: bool = True, size: int = 300) -> "IPython.display.IFrame":
        """Returns the hierarchical representation of this system in HTML format.

        Returns
        -------
        IPython.display.IFrame
            IFrame to the HTML formatted representation
        """
        from cosapp.tools.views.d3js import to_d3
        return to_d3(self, show, size)

    def to_html(self, filename: str, embeddable: bool = False) -> None:
        """Save the `System` as HTML using vis.JS library.

        Parameters
        ----------
        filename : str
            Filename to write to
        embeddable: bool, optional
            Is the HTML to be embedded in an existing page? Default: False
        """
        from cosapp.tools.views.visjs import to_visjs
        to_visjs(self, filename, embeddable)

    def _repr_html_(self) -> str:
        """Returns the representation of this system in HTML format.

        Returns
        -------
        str
            HTML formatted representation
        """
        # TODO unit tests
        temp_name = f"{self.name}.html"
        self.to_html(temp_name)
        from IPython.display import IFrame
        return IFrame(temp_name, "810px", "650px")._repr_html_()

    def _repr_markdown_(self) -> str:
        """Returns the representation of this system attributes in Markdown format.

        Returns
        -------
        str
            Markdown formatted representation
        """
        from cosapp.tools.views.markdown import system_to_md
        return system_to_md(self)

    def make_surrogate(self,
        data_in: Union[pandas.DataFrame, Dict[str, List[float]]],
        model = FloatKrigingSurrogate,
        activate = True,
        data_out: Optional[Union[pandas.DataFrame, Dict[str, Any]]] = None,
        postsynch: Union[str, List[str]] = '*',
        *args, **kwargs) -> SystemSurrogate:
        """
        Creates a surrogate model superseding the normal behaviour of `compute()`.
        The surrogate model is trained from datasets `data_in` and `data_out`,
        given as `pandas.DataFrame` objects, or dictionnaries interpretable as so.

        If no output data are provided (default), they are computed as the response
        of system to design-of-experiment `data_in`.

        Parameters
        ----------
        - data_in: pandas.DataFrame or Dict[str, List[float]]
            Design of experiments for variables in dataframe columns or dict keys.
            Column/key names must match variable contextual names, as in 'wing.geom.length', e.g.
        - model: type
            Model class, implementing methods `train(x, y)` and `predict(x)`, where x and y are 1D arrays.
            Default is `cosapp.systems.surrogate_models.kriging.FloatKrigingSurrogate`.
        - activate: bool, optional
            Boolean determining whether or not surrogate model should be activated once created.
            Default is `True`.
        - data_out: pandas.DataFrame or Dict[str, List[float]], optional
            Y-dataset used for surrogate model training. Default is None (dataset is computed).
        - postsynch: str or List[str]
            List of output variable names to synchronize after each surrogate model execution.
            Default is '*', meaning all outputs are post-synchronized.
        - *args, **kwargs: Any
            Additional parameters passed to `model` constructor.

        Returns:
        --------
        SystemSurrogate
            System surrogate attached to the system.
        """
        check_arg(data_in, 'data_in', (pandas.DataFrame, dict))
        surrogate = SystemSurrogate(self, data_in, model, data_out, postsynch, *args, **kwargs)
        self.__set_surrogate(surrogate, activate)
        return surrogate

    def __set_surrogate(self, surrogate: SystemSurrogate, activate=True) -> None:
        """Private setter for surrogate model"""
        if self._meta is not None:
            warnings.warn(f"Existing surrogate model of {self.name} has been overwritten by a new one.")
        self._meta = surrogate
        self.active_surrogate = activate
    
    def load_surrogate(self, filename: str, activate=True) -> None:
        """
        Loads a surrogate model from a binary file created by `dump_surrogate`.

        Parameters
        ----------
        - filename: str
            Destination file name.
        - activate: bool, optional
            Boolean determining whether or not surrogate model should be activated once loaded.
            Default is `True`.
        """
        surrogate = SystemSurrogate.load(self, filename)
        self.__set_surrogate(surrogate, activate)

    def dump_surrogate(self, filename: str) -> None:
        """Dumps system surrogate model (if any) into a binary file."""
        if self._meta is None:
            raise AttributeError(f"{self.name!r} has no surrogate model")
        self._meta.dump(filename)
    
    @property
    def active_surrogate(self) -> bool:
        """bool: True if surrogate model is activated, False otherwise."""
        return self._meta is not None and self._meta_active_status

    @active_surrogate.setter
    def active_surrogate(self, activated: bool) -> None:
        """Activation boolean setter for surrogate model."""
        check_arg(activated, "active_surrogate", bool)
        if self._meta is None:
            raise AttributeError(
                "Can't set `active_surrogate` if no surrogate model has been created"
                " by either `make_surrogate` or `load_surrogate`."
            )

        if activated == self._meta_active_status:
            return

        if activated:
            # Deactivate owner, children, and all drivers
            self._set_recursive_active_status(False)
            # Reactivate owner after upper recursion
            self._active = True
            self._meta_active_status = True
        
        else:
            self._set_recursive_active_status(True)
            self._meta_active_status = False

    @property
    def has_surrogate(self) -> bool:
        """bool: True if system has a surrogate model (even if inactive), False otherwise."""
        return self._meta is not None

    def _set_recursive_active_status(self, active_status : bool) -> None:
        #TODO save and recover drivers original status
        logger.debug(f"Starting recursive active status modifying of {self.name}, status to be set is {active_status}")
        self._active = active_status
        for driver in self.drivers.values():
            logger.debug(f"Targeted driver for driver recursive deactivate is {driver}")
            driver._set_children_active_status(active_status)
        for child in self.children.values():
            logger.debug(f"Targeted child for system recursive deactivate is {child}")
            child._set_recursive_active_status(active_status)
            

class IterativeConnector(System):
    """Generate residues on close loop connector to mathematical close the equations set.

    The replaced connector is presumed to be removed prior to the construction of a
    `IterativeConnector`.

    Parameters
    ----------
    connector : Connector
        Connector to replace

    Attributes
    ----------
    _sink : ExtensiblePort
        Original connector port sink
    _source : ExtensiblePort
        Original connector port source
    _mapping : Dict[str, str]
        Variable names mapping between source and sink
    """

    GUESS = "guess"
    RESULT = "result"

    # TODO check and complete documentation

    def __init__(self, connector: Connector) -> None:
        """`IterativeConnector` constructor."""
        check_arg(connector, 'connector', Connector)
        sink, source = connector.sink, connector.source
        variables = connector.variable_mapping

        if sink.direction != PortType.IN:
            raise ValueError(
                f"Connector {connector.name!r} cannot be opened.\n"
                "IterativeConnector can only be inserted if the sink is an input."
            )
        # Check that all variables are of numerical types
        for name in variables:
            if not is_numerical(sink[name]):
                raise TypeError(
                    "Cannot open connector {!r} to solve the system, as variable {!r} is non-numerical; got {}.".format(
                        connector.name, name, sink[name]))

        super().__init__(connector.name)

        object.__setattr__(self, '_sink', sink)
        object.__setattr__(self, '_source', source)
        object.__setattr__(self, '_mapping', variables)

        # Create ports
        if isinstance(sink, Port):
            sink_copy = sink.copy(name=IterativeConnector.GUESS, direction=PortType.IN)
        else:
            sink_copy = ExtensiblePort(IterativeConnector.GUESS, PortType.IN)
            for name in variables:
                details = sink.get_details(name)
                sink_copy.add_variable(name, getattr(sink, name), unit=details.unit, dtype=details.dtype)
        self._add_port(sink_copy)

        if isinstance(source, Port):
            source_copy = source.copy(name=IterativeConnector.RESULT, direction=PortType.IN)
        else:
            source_copy = ExtensiblePort(IterativeConnector.RESULT, PortType.IN)
            for name in variables.values():
                details = source.get_details(name)
                source_copy.add_variable(name, getattr(source, name), unit=details.unit, dtype=details.dtype)
        self._add_port(source_copy)

        # Set the mathematical problem
        sink_unknowns = sink.owner.unknowns
        self._locked = False  # add_unknown and add_equation not allow outside `setup`
        for to_, from_ in self._mapping.items():
            sink_name = f"{sink.name}.{to_}"
            unknown = f"{IterativeConnector.GUESS}.{to_}"
            if sink_name in sink_unknowns:  # Copy numerical feature from user specifications
                self.add_unknown(
                    unknown, 
                    max_abs_step=sink_unknowns[sink_name].max_abs_step,
                    max_rel_step=sink_unknowns[sink_name].max_rel_step,
                    upper_bound=sink_unknowns[sink_name].upper_bound,
                    lower_bound=sink_unknowns[sink_name].lower_bound,
                )
            else:
                self.add_unknown(unknown)
            rhs = f"{IterativeConnector.RESULT}.{from_}"
            self.add_equation(
                f"{unknown} == {rhs}",
                name=f"iterative_{to_}",
                reference=1.,
            )
        self._locked = True

    def get_connection(self) -> Tuple[ExtensiblePort, ExtensiblePort, Dict[str, str]]:
        """Tuple[ExtensiblePort, ExtensiblePort, Dict[str, str]] : Returns the connection elements.
        
        The first element is the sink port, the second the source port and the latest is the 
        variable mapping.
        """
        return (self._sink, self._source, self._mapping)

    def set_dirty(self, direction: PortType) -> None:
        """Set to dirty ports of a certain direction.

        Parameters
        ----------
        direction : PortType
            Direction to set
        """
        # TODO Do not propagate to parent otherwise this will kill the benefit
        # of clean-dirty => one more reason to move it at port level
        self._is_clean[direction] = False
