"""Dictionary list :class:`Dlist`

Is no more than a list of dictionaries that optionally can have 
an ``id`` field. The interesting part is in the methods for operation
with this object in a database-like way.

.. note::

   In the dictionaries, ``key`` must be *string* type and ``val`` 
   can only be a *string* or a *dictionary*

.. warning::

   Limitations in key names:

   * Reserved key: ``complement``
   * Reserved value: ``__dict__``

**Nested Dictionaries:** if a ``value`` is type *dict*, the keys of the subdict can be recalled 
as keys of the main dictionary using ``__`` (double underscore) as separator, similar to 
Django's ORM lookup syntax. This is the ``flatten-version`` of the original dictionary. 
The separator is configurable via ``charSep``.

Example::

    {'k1': 'v1', 'k2': {'a': 'va', 'b': 'vb', 'c': {'qq': 'pi2'}}, 'k3': {'A': 'vA'}}
    # the flatten version is: 
    {'k1': 'v1', 'k2__a': 'va', 'k2__b': 'vb', 'k2__c__qq': 'pi2', 'k3__A': 'vA'}

"""

from fnmatch import fnmatch  # glob expressions
from copy import deepcopy
from .helpers import flattenDict, structDict, merge_dicts, getDitem, chars


def _fmatch(value, pattern):
    """fnmatch wrapper: if *value* is a list, return True when any element matches."""
    if isinstance(value, list):
        return any(fnmatch(str(v), pattern) for v in value)
    return fnmatch(str(value), str(pattern))


class Dlist:
    """Dictionary List main class

    Parameters:
        dl (dict list, default=empty): List of dictionaries
        id_ (str, default=''): id keyword

    Usage::

      # Empty object
      d = Dlist()

      # Without index
      data = [{'name':'willy', 'size':'M'},
              {'foo':'bar'}
             ]
      d = Dlist(data)

      # With index (all fields must have different id fields)
      data = [{'name':'willy', 'size':'M', 'No':1},
              {'foo':'bar', 'No': 3}
             ]
      d = Dlist(data, 'No') #(the number 3 is converted to ``str`` type)

    ========== ====== =======================================
    Attributes type   .
    ========== ====== =======================================
    **l**      *list* list of dicts
    **id**     *str*  ``id`` field name ('' if not indexed) 
    ========== ====== =======================================
    """

    def __init__(self, dl=[], id_=''):
        self.l = deepcopy(list(dl))
        self.id = id_
        self.charSep = '__'
        
        if self.id != '' and len(self.l) != 0:
            # Check if duplicate and enough
            if len(self.vals(self.id)) != len(self):
                raise ValueError('Dlist: Duplicate or not declared all ids')
        
        # Ensure string type for scalar fields (preserve lists)
        for i, field in enumerate(self.l):
            dic = flattenDict(field, charSep=self.charSep)
            for k in dic:
                if not isinstance(dic[k], list):
                    dic[k] = str(dic[k])
            self.l[i] = structDict(dic, charSep=self.charSep)

    
    def sort(self, key=None, reverse=False):
        '''Sorts the list in place

        Parameters:
           key (str|list): key or list of keys to sort by (default: id field).
               When a list, sorts by the first key, then by the second among
               ties, and so on.  ``__`` syntax available.
           reverse (bool): reverse order

        Returns:
           self (for chaining)

        Examples::

            >>> d.sort('name')
            >>> d.sort(['type', 'name'])   # primary by type, secondary by name
        '''
        if key is None:
            key = self.id
        if key == '' or key == []:
            return self
        if isinstance(key, list):
            self.l = sorted(self.l,
                            key=lambda x: tuple(getDitem(x, k, charSep=self.charSep) for k in key),
                            reverse=reverse)
        else:
            self.l = sorted(self.l, key=lambda x: getDitem(x, key, charSep=self.charSep), reverse=reverse)
        return self

    ###################################################################
    ### Iterators

    def __iter__(self):
        return iter(self.l)
    
    def __len__(self):
        self.l = list(self.l)
        return len(self.l)
       
    def index(self, value):
        '''Finds the index number in list, given id or field number

        Parameters:
           value (id or dict):  searching *id* or complete *dict* field

        The dict case finds by exact full dict match::

            D = Dlist([{'id': '1', 'b': '2'}, {'id': '3', 'idx': '4'}], id_='id')
            D.index({'id': '3', 'idx': '4'})  # returns 1

        '''
        match value:
            case str() if self.id != '':
                for i, dd in enumerate(self.l):
                    if dd[self.id] == value:
                        return i
                return -1
            case dict():
                for i, dd in enumerate(self.l):
                    if dd == value:
                        return i
                return -1
            case _:
                raise ValueError('Dlist: incorrect type')
               
    def __getitem__(self, index):    
        """Method for getting item ``dl[x]``

        ==================== ==========
        Call                 Returns
        ==================== ==========
        dl [*index*]         -> dict  
        dl [*id*]            -> dict  
        dl [*list.of.index*] -> dlist  
        dl [*list.of.ids*]   -> dlist
        ==================== ==========

        """
        match index:
            case int():
                return self.l[index]
            case str() if self.id != '':
                try:
                    return self.filter(**{self.id: index}).l[0]
                except:
                    return Dlist([])
            case str():
                raise ValueError('Dlist: string index in dlist without id')
            case list() if len(index) == 0:
                return Dlist([], self.id)
            case list():
                if isinstance(index[0], int) or (isinstance(index[0], str) and self.id != ''):
                    return Dlist([self[i] for i in index], self.id)
                raise ValueError('Dlist: not a valid index')
                 
    def __setitem__(self, index, value):
        '''Method for setting item ``dl[x]``

        ====================  =====================
        Assign                Value
        ====================  =====================
        dl [*index*]          = dict  
        dl [*id*]             = dict 
        dl [*list.of.index*]  = dict, list.of.dicts  
        dl [*list.of.ids*]    = dict, list.of.dicts
        dl [*dlist*]          = dict 
        ====================  =====================

        Assignment merges dictionaries, accepting changes in declared
        keys and supporting nested structure syntax.
        '''
        match index:
            case int():
                if not isinstance(value, dict):
                    raise ValueError('Dlist: assignment of one element must be dict')
                self.l[index] = self.merge(self.l[index], value)

            case str() if self.id != '':
                if not isinstance(value, dict):
                    raise ValueError('Dlist: assignment of one element must be dict')
                i0 = self.index(index)
                if i0 == -1:
                    raise ValueError('Dlist: index id not in dlist')
                self.l[i0] = self.merge(self.l[i0], value)

            case str():
                raise ValueError('Dlist: string index in dlist without id')

            case list():
                match value:
                    case dict():
                        value = [value] * len(index)
                    case list() if len(index) != len(value):
                        raise ValueError('Dlist: list of index must have same length as list of values')
                    case list():
                        pass
                    case _:
                        raise ValueError('Dlist: only can assign dict or list to Dlist(list)')
                for i, v in zip(index, value):
                    self[i] = v

            case Dlist() if isinstance(value, dict):
                if self.id != index.id:
                    raise ValueError('Dlist: in dlist1[dlist2] both dlist must have same id')
                for d in index:
                    lookup = d[self.id] if self.id != '' else d
                    i0 = self.index(lookup)
                    if i0 != -1:
                        self.l[i0] = self.merge(self.l[i0], value)

            case Dlist():
                raise ValueError('Dlist: only can assign dict to Dlist(Dlist)')

            case _:
                raise ValueError('Dlist: incorrect index type')
            
    ###################################################################
    ### Queries
        
    def filter(self, complement=False, **query):
        '''Filters a :class:`Dlist` by attributes values

        * **query** is a dictionary with key:value to select. Accept subestructures with ``__`` notation
        * if ``complement=True`` returns two :class:Dlist the filtered and the complement
        * Returns a :class:`Dlist`
        * ``key:True``, selects dicts with that ``key`` present, irrespective of its value
        * ``key:False``, selects dicts without that ``key``
        * if ``value`` begins with ``'-'``, excludes that value in result
        * Allow ``glob`` syntax in ``values``
        * If a field value is a **list**, matches when the query value is contained in that list
        * Character ``'_'`` at the end of ``key`` is ignored (to avoid collision with reserved names)

        Usage::

           # select for type and dir
           dlf = dl.filter(type='video',dir='mystuff')

           # select not video and glob
           dlf = dl.filter(type='-video',dir='my*')
           
           # select items without declared sync
           dlf = dl.filter(sync=False)

           # list membership: matches items where 'photo' is in tags list
           dlf = dl.filter(tags='photo')

        '''
        keys = list(query.keys())
        
        ### no query
        if len(keys) == 0: 
            if not complement:
                return self
            else:
                return self, Dlist([])
        
        ### cleans end underscores at the end of keys
        for k in keys:
            if k.endswith('_'):
                query[k[:-1]] = query[k]
                del query[k]
        keys = list(query.keys())        
      
        ### key negation
        noKeys = []                  # <-- list of absent keys
        for k in keys:
            if query[k] == False:
                noKeys.append(k)
                query[k] = ''

        ### key affirmation
        yesKeys = []                  # <-- list of required keys
        for k in keys:
            if query[k] == True:
                yesKeys.append(k)
                query[k] = ''

        ### value negation
        noVals = []                  # <-- list of keys with negated values
        for k in keys:
            if k not in noKeys and k not in yesKeys:
                if query[k].startswith('-'):  
                    noVals.append(k)
                    query[k] = query[k][1:]        
        
        ### Función de filtrado - d is a dict
        def filterFunction(d):
            D = flattenDict(d, charSep=self.charSep)

            xor = lambda p, q: (p and not q) or (not p and q)                      ## just logic
            
            isGood = map(lambda k: not((k in yesKeys) and (not (k in D))) and   ## yeskeys
                         xor((k in D), k in noKeys) and                           ## nokeys
                         (((k in noKeys) or (k in yesKeys)) or                   # only check value if ...
                                   xor(_fmatch(D[k], query[k]), k in noVals)   ## val 
                         ), keys)
            
            return all(list(isGood))
        
        out = Dlist(filter(filterFunction, self.l), self.id)

        if not complement:
            return out
        else:
            return out, self - out
    
    def vals(self, key, count=False):
        '''Set of values for a given ``key``
        
        Only returns string values, not the nested dicts
        
        Parameters:
           key (str): key to search. ``__`` syntax available
           count (bool): returns the number of occurrences or not

        * the sorting is by appearance
        * ``count=False``: returns a ``list``
        * ``count=True``: returns a ``dict`` with the number of occurrences
        
        '''
        raw = map(lambda x: getDitem(x, key, charSep=self.charSep), self.l)
        raw = list(filter(lambda x: x != {}, raw))          # <- eliminates {}
        # Expand list values into individual elements
        vals = []
        for v in raw:
            if isinstance(v, list):
                vals.extend(v)
            else:
                vals.append(v)
        
        # unrepeated and sorted list of values
        vs = self._dElements(vals)
        
        if not count: 
            return vs
        else:
            out = {}
            for v in vs:
                out[v] = vals.count(v)
            return out
            
    def keys(self, root='', All=False, count=False, same=False):
        """Set of keys available

        Parameters:
           root (str): base key in the dict tree. Nested ``__`` syntax
           All (bool): if True, keys must be present in all fields
           count (bool): returns the number of occurrences of each key
           same (bool): if True, outputs keys with only one value, can be combined with `All`

        * sorted alphabetically
        * ``count=False``: returns a ``list``
        * ``count=True`` : returns a ``dict`` with the number of occurrences
        * ``All  =True`` : outputs only keys presents in all fields

        """
        if same:
            # performs standard search
            ksdic = self.keys(root, All, count)
            if count:
                ks = ksdic.keys()
            else:
                ks = ksdic
            # cleans root
            if root != '' and not root.endswith(self.charSep):
                root_ = root + self.charSep
            else:
                root_ = root

            # returns True if there is a unique value in key    
            def oneValue(k):
                kk = f'{root_}{k}'
                v = self.vals(kk, count=True)
                if len(v) == 1:
                    if not All:
                        return True
                    else:
                        if list(v.values())[0] == len(self.l):
                            return True
                        else: 
                            return False
                else:
                    return False
  
            sameks = list(filter(oneValue, ks)) 
            if not count:
                return sameks
            else:
                out = {}
                for k in sameks:
                    out[k] = ksdic[k]
                return out

        # standard routine
        ksa = []
        for item in self.l:
            ritem = getDitem(item, root, charSep=self.charSep)
            try:
                ksa.extend(list(ritem.keys()))
            except:
                pass
            
        ks = sorted(list(set(ksa)))
        
        if not All:
            if not count:
                return ks
            else:
                out = {}
                for k in ks:
                    out[k] = ksa.count(k)
                return out
        else:
            # All=True
            # function returns True if k is a key in all fields
            
            allTest = lambda k: all(list(map(lambda x: k in getDitem(x, root, charSep=self.charSep), self.l)))
            allks = list(filter(allTest, ks))
            if not count:
                return allks
            else:
                out = {}
                for k in allks:
                    out[k] = ksa.count(k)
                return out

    def tree(self, root='', top=3, depth=None, printTree=True):
        '''Tree representation of the Dlist structure for interactive exploration

        Parameters:
           root (str): base key in the dict tree. Nested ``__`` syntax
           top (int): number of most frequent values to show (default: 3)
           depth (int|None): max depth of branches to expand (None = unlimited)

        Returns:
           str: tree string ready for terminal display

        The leaf format is::

            key un/kn [val1 (vn1), val2 (vn2), ...]

        where *un* = unique values, *kn* = records with this key,
        *vn* = occurrences of each value.

        When *depth* is set, branches deeper than *depth* are shown as
        node names without expanding or showing value stats.

        Example::

            >>> print(d.tree())
            Dlist (3 items, id='name')
            ├── name 3/3 ['alice' (1), 'bob' (1), 'charlie' (1)]
            ├── age 3/3 ['30' (1), '25' (1), '28' (1)]
            └── info
                ├── city 2/3 ['london' (2), 'paris' (1)]
                └── job 3/3 ['dev' (1), 'pm' (1), 'qa' (1)]
        '''
        # Flatten all items once — work on flat dicts throughout
        sep = self.charSep
        flat_items = [flattenDict(d, charSep=sep) for d in self.l]

        def _keys_at(items, prefix):
            """Collect unique sub-keys under prefix from flat items."""
            if prefix:
                pfx = prefix + sep
                plen = len(pfx)
            else:
                pfx = ''
                plen = 0
            ks = set()
            for D in items:
                for k in D:
                    if plen == 0:
                        ks.add(k.split(sep)[0])
                    elif k.startswith(pfx):
                        ks.add(k[plen:].split(sep)[0])
            return sorted(ks)

        def _partition(items, full_key):
            """Group flat items by value at full_key. O(n) single pass."""
            groups = {}
            for D in items:
                if full_key in D:
                    v = D[full_key]
                    h = tuple(v) if isinstance(v, list) else v
                else:
                    h = None
                groups.setdefault(h, []).append(D)
            return groups

        def _branch(items, root_prefix, is_last_map, current_depth=0):
            lines = []
            ks = _keys_at(items, root_prefix)

            for idx, k in enumerate(ks):
                is_last = (idx == len(ks) - 1)
                connector = chars.lr + chars.hl + chars.hl if is_last else chars.tr + chars.hl + chars.hl
                prefix = ''
                for lvl_last in is_last_map:
                    prefix += '    ' if lvl_last else chars.vl + '   '

                full_key = f'{root_prefix}{sep}{k}' if root_prefix else k
                children = _keys_at(items, full_key)

                if children:
                    if depth is not None and current_depth >= depth:
                        lines.append(f'{prefix}{connector} {k}   \t...')
                    else:
                        lines.append(f'{prefix}{connector} {k}')
                        # Collect items that have this branch
                        pfx = full_key + sep
                        sub_items = [D for D in items if any(k.startswith(pfx) for k in D)]
                        lines.extend(_branch(sub_items, full_key, is_last_map + [is_last], current_depth + 1))
                else:
                    # Leaf: partition gives counts in one pass
                    parts = _partition(items, full_key)
                    kn = sum(len(v) for v in parts.values())
                    un = len(parts) - (None in parts)
                    counts = sorted(
                        [(v, len(lst)) for v, lst in parts.items() if v is not None],
                        key=lambda x: -x[1])[:top]

                    fmts = [f"'{', '.join(v)}' ({c})" if isinstance(v, tuple)
                            else f"'{v}' ({c})" for v, c in counts]
                    if un > top:
                        val_str = ', '.join(fmts) + ', ...'
                    else:
                        val_str = ', '.join(fmts)

                    lines.append(f"{prefix}{connector} {k:7s}\t{un}/{kn} [{val_str}]")

            return lines

        # Header
        header = f'Dlist ({len(self.l)} items, id=\'{self.id}\')'
        lines = [header]
        if root != '':
            lines.append(root)
        lines.extend(_branch(flat_items, root, []))
        out = '\n'.join(lines)
        if printTree:
            print(out)
            return
        else:
            return out

    def _dElements(self, lst, withDict=False):
        '''Elements in lst (struct dict)'''

        ou = []
        for item in lst:
            if item not in ou:
                ou.append(item)
        ousS = sorted(list(filter(lambda x: isinstance(x, str), ou)))

        if withDict:
            ousD = list(filter(lambda x: isinstance(x, dict), ou))
            return ousS + ousD
        else:
            return ousS

    def map(self, f, inputL=[], outputL=[], args=(), keyargs={}, query={}):
        """Applies a Python function to each field with required keys

        Parameters:
           f (function): function to apply
           inputL (list): list of keys for input (all must be present to apply)
           outputL(list): list of keys for storing the output. If values exist, overrides
           args   (tuple): extra args
           keyargs (dict): extra keyargs
           query (dict): apply only on fields that satisfies the query. See :meth:`filter`

        Returns:
           Dlist: new Dlist with all records (modified + untouched)

        * applies ``f(field[inputL],*args,**keyargs) -> field[outputL]``
        * ``f`` can return a list, a tuple, or a single value (when ``len(outputL)==1``)
        * if ``f`` returns None, the record is kept unchanged
        * non-mutating: returns a new :class:`Dlist`, original is untouched
        * in the input and output list the nested ``__`` syntax is available.

        """
        # Build the filter function from query (reuse filter's logic)
        if query:
            keys = list(query.keys())
            # clean trailing underscores
            for k in list(keys):
                if k.endswith('_'):
                    query[k[:-1]] = query[k]
                    del query[k]
            keys = list(query.keys())

            noKeys = [k for k in keys if query[k] == False]
            for k in noKeys: query[k] = ''
            yesKeys = [k for k in keys if query[k] == True]
            for k in yesKeys: query[k] = ''
            noVals = [k for k in keys if k not in noKeys and k not in yesKeys and query[k].startswith('-')]
            for k in noVals: query[k] = query[k][1:]

            def _matches(d):
                D = flattenDict(d, charSep=self.charSep)
                xor = lambda p, q: (p and not q) or (not p and q)
                isGood = map(lambda k: not((k in yesKeys) and (not (k in D))) and
                             xor((k in D), k in noKeys) and
                             (((k in noKeys) or (k in yesKeys)) or
                                       xor(_fmatch(D[k], query[k]), k in noVals)
                             ), keys)
                return all(list(isGood))
        else:
            _matches = lambda d: True

        result = []
        for d in self.l:
            if not _matches(d):
                result.append(deepcopy(d))
                continue

            # Input args
            IL = [getDitem(d, k, charSep=self.charSep) for k in inputL]
            if {} in IL:
                result.append(deepcopy(d))
                continue

            # Function
            out = f(*IL, *args, **keyargs)
            if out is None:
                result.append(deepcopy(d))
                continue

            # Normalize: single value -> list when one output key
            if len(outputL) == 1 and not isinstance(out, (list, tuple)):
                out = [out]

            # Output: merge new keys into copy of original
            ouD = {}
            for ok, ov in zip(outputL, out):
                ouD[ok] = ov
            result.append(merge_dicts(d, ouD, over=True, charSep=self.charSep))

        return Dlist(result, self.id)

    def partition(self, key):
        """Makes a partition of evidence based on key values

        Parameters:
           key (str): extended key for partition

        Returns:
            {keval1:dlist1, ...}
            the partition where key is not defined goes to the keyval=None

        """

        out = {}
        cats = {}                   # hashable_key → original value

        # loads dic in lists - scan of ev
        for dic in self:
            D = flattenDict(dic, charSep=self.charSep)
            if key in D:
                cat_i = D[key]
            else:
                cat_i = None

            # Make hashable key for dict lookup
            h = tuple(cat_i) if isinstance(cat_i, list) else cat_i

            if h in cats:
                out[h].append(dic)
            else:
                cats[h] = cat_i
                out[h] = [dic]

        # make dlists
        out = {h: Dlist(out[h], id_=self.id) for h in cats}
        
        return out

    def assign(self, query={}, **kwargs):
        """Assigns ``key=value`` for every field matching query

        Parameters:
            query (dict, optional): select fields to apply. See :meth:`filter`
            **kwargs: key=value pairs to assign

        Returns:
           Dlist: new Dlist with all records (modified + untouched)

        Usage::

          # merges {'cat':'new'} to all fields
          d2 = d.assign(cat='new')

          # merges to fields with name=w*
          d2 = d.assign(query={'name':'w*'}, cat='new')

        Non-mutating: returns a new :class:`Dlist`, original is untouched.
        Allows nested ``__`` syntax in query and keys.
        """
        v = list(kwargs.values())

        return self.map(lambda: v,
                        [], list(kwargs.keys()),
                        query=query)

    def set_id(self, key, generate=None, digits=0):
        """Set or generate an id field, returning a new Dlist.

        Parameters:
            key (str): Field name to use as id.  If the field already
                exists in the records it is promoted to id.  If
                *generate* is given, a new field is created first.
            generate (str|None): Prefix for auto-generated ids.
                When set, sequential values like ``R01``, ``R02`` …
                are written into *key* before setting it as id.
            digits (int): Number of zero-padded digits in generated
                ids.  ``0`` (default) = auto: one more digit than
                ``len(str(len(data)))``.  E.g. 5 records → 2 digits,
                150 records → 4 digits.

        Returns:
            Dlist: New Dlist with *key* as id

        Raises:
            ValueError: If *key* field has duplicate values
            ValueError: If *generate* is used and *key* already exists
                in any record (use *overwrite=True* to allow)

        Examples::

            >>> d2 = d.set_id('name')
            >>> d2 = d.set_id('id', generate='R')
            >>> d2 = d.set_id('id', generate='E', digits=5)
        """
        import copy

        if generate is not None:
            # Safety: don't silently overwrite an existing field
            if any(key in item for item in self.l):
                raise ValueError(
                    f"set_id: field '{key}' already exists in records. "
                    f"Use a different key name or remove the field first.")
            n = len(self.l)
            if digits <= 0:
                digits = len(str(n)) + 1
            records = []
            for i, item in enumerate(self.l):
                rec = copy.copy(item)
                rec[key] = f'{generate}{i + 1:0{digits}d}'
                records.append(rec)
            return Dlist(records, id_=key)
        else:
            return Dlist([copy.copy(d) for d in self.l], id_=key)

    ###################################################################
    ### Add and Subtraction operands
    
    def __sub__(self, other):
        """Subtraction operand
        
        Subtract two :class:`Dlist`

        * if any of the operands are without ``id`` only removes identical 
          fields
        * with ``id`` removes the field from **dl1** with 
          the same ``id`` irrespective of its contents

        Usage::

           dlSbs = dl1 - dl2

        """

        if (self.id == '') or (other.id == ''):
            ### without id
            ll = filter(lambda d: d not in other, self.l)
            return Dlist(ll)
        else:
            ### with id
            ids = other.vals(other.id)
            ll = filter(lambda d: d[self.id] not in ids, self.l)
            return Dlist(ll, self.id)
      
    def __add__(self, other):
        """Addition operand

        Merges two :class:`Dlist`

        Usage::

           dlSumm = dl1 + dl2

        .. note::
           Asymmetrical operation. In case of conflict, preserves values of first operand

           Operates inside structured dicts

        """
        if (self.id == '') or (other.id == ''):
            ### without id
            return Dlist((self - other).l + other.l)
        else:
            ### with id
            out = []
            ids = other.vals(other.id)
            for ditem in self:
                if ditem[self.id] not in ids:
                    ## other no tiene este id
                    out.append(ditem)
                else:
                    ids.remove(ditem[self.id])
                    ## mezclo dos diccionarios
                    dic_other = other[ditem[self.id]]
                    out.append(self.merge(ditem, dic_other, over=False))
            for i in ids:
                ## other not in self
                dic_other = other.filter(**{other.id: i}).l[0]
                out.append(dic_other)

        return Dlist(out, self.id)

    def __eq__(self, other):
        '''Equal operator

        '''
        dl1 = Dlist(self.l)
        dl2 = Dlist(other.l)
        return len(dl1) == len(dl2) and (dl1 - dl2).l == [] and dl1.id == dl2.id

    def merge(self, d1, d2, over=True):
        '''Merge two dictionaries, considering structure

        Parameters:
           d1 (dict): dictionary_1
           d2 (dict): dictionary_2
           over (bool): if True, accepts override changes in declared keys of **d1**

        .. note::

           The operation is not symmetric, but these sentences are equivalent

           ``self.merge(d1, d2, over=True) <--> self.merge(d2, d1, over=False)``
        '''
        return merge_dicts(d1, d2, over=over, charSep=self.charSep)
    
    ###################################################################
    ### I/O methods

    @staticmethod
    def _format_from_ext(filepath, default='json'):
        """Infer format from file extension, or return *default*."""
        _EXT_MAP = {'.json': 'json', '.xlsx': 'excel'}
        if isinstance(filepath, str):
            import os
            _, ext = os.path.splitext(filepath)
            if ext.lower() in _EXT_MAP:
                return _EXT_MAP[ext.lower()]
        return default

    @classmethod
    def read(cls, source, id_='', format=None, **kwargs):
        """Create a Dlist from an external source

        Parameters:
            source: File path, string, or data structure
            id_ (str): Optional id field name
            format (str|None): Source format ('json', 'excel').
                If ``None``, inferred from the file extension
                (``.xlsx`` → excel, ``.json`` → json).  Falls back
                to ``'json'``.
            **kwargs: Passed to the parser (e.g. ``header_row``,
                ``data_start``, ``sheets`` for Excel)

        Returns:
            Dlist: New Dlist object

        Examples::

            >>> d = Dlist.read('data.json', id_='id')
            >>> d = Dlist.read('report.xlsx')              # auto-detected
            >>> d = Dlist.read('report.xlsx', header_row=3)
            >>> d = Dlist.read('[{"a":1}]')
        """
        if format is None:
            format = cls._format_from_ext(source)
        from .parsers import get_parser
        return get_parser(format).parse(source, id_=id_, **kwargs)

    @classmethod
    def read_pivot(cls, files, pivot='id', base=None):
        """Join multiple JSON files by a shared pivot key using DuckDB

        Each file is a JSON array of objects sharing a common ``pivot``
        field.  The non-pivot columns of every file are merged into a
        single Dlist record via SQL LEFT JOIN.

        Parameters:
            files (dict|str): ``{filepath: key_name}`` mapping, or a
                **glob pattern** (e.g. ``'data/*.json'``).  When a glob
                is given, key names are derived from each file's stem
                with a trailing ``'s'`` stripped (``types.json`` → ``'type'``).
            pivot (str): The join key present in every file (default: ``'id'``).
            base (str|None): File that drives the result set. If ``None``,
                the first file in *files* is used.

        Returns:
            Dlist: Merged Dlist with ``id_=pivot``

        Examples::

            >>> d = Dlist.read_pivot('evidence/*.json')

            >>> d = Dlist.read_pivot({
            ...     'files.json':  'file',
            ...     'types.json':  'type',
            ...     'exifs.json':  'exif',
            ... })
        """
        from .parsers import get_parser
        return get_parser('json').parse_pivot(files, pivot=pivot, base=base)

    def write(self, filepath=None, format=None, ctree=None, **kwargs):
        """Export Dlist to string or save to file

        If ``ctree`` is provided, produces categorized output (one call
        handles both basic and categorized).

        Parameters:
            filepath (str|None): If given, save to file. Otherwise return string.
            format (str|None): Output format ('json', 'ascii', 'md', 'latex',
                'excel').  If ``None``, inferred from *filepath* extension
                (``.xlsx`` → excel, ``.json`` → json).  Falls back to
                ``'json'``.
            ctree (list|None): Category keys for grouped output. When None,
                produces a basic table.
            **kwargs: Passed to the formatter — ``keys``, ``titles``,
                ``width``, ``nCols``, ``page``, etc.

        Returns:
            str|None: Formatted string if filepath is None

        Examples::

            >>> s = d.write()                                    # JSON string
            >>> d.write('output.xlsx', keys=['id', 'name'])      # auto-detected
            >>> s = d.write(format='ascii', keys=['id', 'name']) # explicit
            >>> d.write('out.json')                              # save to file
        """
        if format is None:
            format = self._format_from_ext(filepath)
        from .formatters import get_formatter
        fmt = get_formatter(format)

        if ctree is not None:
            keys = kwargs.pop('keys', [])
            titles = kwargs.pop('titles', {})
            if hasattr(fmt, 'save_categorized') and filepath is not None:
                fmt.save_categorized(self, ctree, keys, filepath=filepath,
                                     titles=titles, **kwargs)
                return None
            content = fmt.format_categorized(self, ctree, keys,
                                             titles=titles, **kwargs)
        else:
            if filepath is not None:
                fmt.save(self, filepath, **kwargs)
                return None
            content = fmt.format(self, **kwargs)

        if filepath is not None:
            with open(filepath, 'w', encoding='utf-8') as f:
                f.write(content)
        else:
            return content

    def write_categorized(self, ctree, keys, filepath=None, format='json',
                          titles={}, **kwargs):
        """Export Dlist with categorized grouping

        .. deprecated:: Use ``write(ctree=..., keys=...)`` instead.
        """
        return self.write(filepath=filepath, format=format, ctree=ctree,
                          keys=keys, titles=titles, **kwargs)

    ###################################################################
    ### String representation (minimal, no formatters needed)

    def __str__(self):
        return f"Dlist({len(self.l)} items, id='{self.id}')"

    def __repr__(self):
        out = f"Dlist(\n"
        for item in self.l:
            out += f"   {item!r}, \n"
        out += f"id_='{self.id}')"
        return out
