"""
SPDX-License-Identifier: Apache-2.0
Copyright Contributors to the ODPi Egeria project.

Utilities for rendering Egeria elements into various output formats
such as DICT, LIST, Markdown (including Mermaid), and basic HTML.

Exceptions
----------
Functions in this module may raise the following exceptions:
- ValueError: Unsupported or inconsistent output format requests (e.g., LIST/TABLE
  without the required columns); invalid option combinations.
- KeyError: Missing expected keys from provided elements or report specifications.
- TypeError: Incorrect input types (e.g., non-list where a list of dicts is required).
- RuntimeError: Failures in Markdown/HTML conversion or Mermaid rendering helpers.
"""

import copy
from datetime import datetime
import re
from typing import Any, Callable, Dict, List, Optional, Tuple, Union

from pyegeria.core._globals import MERMAID_GRAPH_TITLES, MERMAID_GRAPHS
from pyegeria.core.utils import (camel_to_title_case)
from markdown_it import MarkdownIt
from rich.console import Console
from loguru import logger

from pyegeria.view.mermaid_utilities import construct_mermaid_web
from pyegeria.view.base_report_formats import select_report_format, MD_SEPARATOR, get_report_spec_match
from pyegeria.models import to_camel_case

# Flag to control whether Mermaid graphs are normalized for broad compatibility (e.g. Obsidian, PyCharm).
# Egeria often produces Mermaid output using newer syntax features (like @{ shape: ... })
# or with YAML frontmatter that some Markdown renderers do not yet support.
#
# Rules applied when NORMALIZE_MERMAID = True:
# 1. Line Endings: Standardize on \n.
# 2. Frontmatter: Remove leading --- blocks (YAML frontmatter) which can break older parsers.
# 3. Label Escaping: Literal newlines within double-quoted labels are escaped to \n to prevent
#    parsing errors in environments like Obsidian.
# 4. Shape Syntax: Converts @{ shape: type, label: "text" } syntax to standard Mermaid shapes:
#    - 'rounded' or 'stadium' -> (text)
#    - 'diamond' or 'decision' -> {text}
#    - 'circle' -> ((text))
#    - 'hexagon' -> {{text}}
#    - Others (e.g. 'doc') -> [text] (standard square box)
#
# Set to False to preserve the native Egeria output for modern Mermaid renderers.
NORMALIZE_MERMAID = False

"""
Note on select_report_spec function:

This function and related data structures have been moved back to _output_formats.py.
Please import select_report_spec from pyegeria._output_formats instead of from this module.
"""
# Todo - put this back after testing
# console = Console(width=settings.Environment.console_width)
console = Console(width=300)


def _extract_referenceable_properties(element: dict[str, Any]) -> dict[str, Any]:
    # Get general header attributes
    guid = element.get('elementHeader', {}).get("guid", None)
    if guid is None:
        return {}
    metadata_collection_id = element['elementHeader']['origin'].get("homeMetadataCollectionId", None)
    metadata_collection_name = element['elementHeader']['origin'].get("homeMetadataCollectionName", None)
    origin_category = element['elementHeader'].get("origin_category", None)
    created_by = element['elementHeader']["versions"].get("createdBy", None)
    create_time = element['elementHeader']["versions"].get("createTime", None)
    updated_by = element['elementHeader']["versions"].get("updatedBy", None)
    version = element['elementHeader']["versions"].get("version", None)
    type_name = element['elementHeader']["type"].get("typeName", None)
    classifications = element['elementHeader'].get("classifications", [])
    effective_from = element['elementHeader'].get("effectiveFrom", None)
    effective_to = element['elementHeader'].get("effectiveTo", None)
    status = element['elementHeader'].get("status", None)
    # Get attributes from properties
    properties = element.get('properties',None)
    if properties:
        url = properties.get('url', None)
        display_name = properties.get("displayName","")
        description = properties.get("description", "") or ""
        qualified_name = properties.get("qualifiedName", "") or ""
        category = properties.get("category", "") or ""
        version_identifier = properties.get("versionIdentifier", "") or ""
        additional_properties = properties.get("additionalProperties", {}) or {}
        extended_properties = properties.get("extendedProperties", {}) or {}
        contentStatus = properties.get("contentStatus", "")
        activityStatus = properties.get("activityStatus", "")
        deploymentStatus = properties.get("deploymentStatus", "")
    else:
        display_name = element.get('displayName', "")
        description = element.get('description', "")
        qualified_name = element.get('qualifiedName', "")
        request_type = element.get('requestType', "")
        requester_user_id = element.get('requesterUserId', "")
        governance_engine_name = element.get('governanceEngineName', "")
        domain_identifier = element.get('domainIdentifier', 0)
        action_status = element.get('actionStatus', "")
        requested_time = element.get('requestedTime', None)
        start_time = element.get('startTime', None)
        url = None
        category = None
        version_identifier = None
        additional_properties = None
        extended_properties = None
        contentStatus = None
        activityStatus = None
        deploymentStatus = None


        description = None
        qualified_name = None


    return {
        "GUID": guid,
        "metadata_collection_id": metadata_collection_id,
        "metadata_collection_name": metadata_collection_name,
        "origin_category": origin_category,
        "created_by": created_by,
        "create_time": create_time,
        "updated_by": updated_by,
        "version": version,
        "type_name": type_name,
        "classifications": classifications,
        "element_status": status,
        "url": url,
        "display_name": display_name,
        "description": description,
        "qualified_name": qualified_name,
        "category": category,
        "version_identifier": version_identifier,
        "additional_properties": additional_properties,
        "extended_properties": extended_properties,
        "effective_from": effective_from,
        "effective_to": effective_to,
        "content_status": contentStatus,
        "activity_status": activityStatus,
        "deployment_status": deploymentStatus,
        }






def markdown_to_html(markdown_text: str) -> str:
    """
    Convert markdown text to HTML, with special handling for mermaid code blocks.

    Args:
        markdown_text: The markdown text to convert

    Returns:
        HTML string
    """
    # Initialize markdown-it
    md = MarkdownIt()

    # Find all mermaid code blocks
    mermaid_blocks = re.findall(r'```mermaid\n(.*?)\n```', markdown_text, re.DOTALL)

    # Replace each mermaid block with a placeholder
    placeholders = []
    for i, block in enumerate(mermaid_blocks):
        placeholder = f"MERMAID_PLACEHOLDER_{i}"
        markdown_text = markdown_text.replace(f"```mermaid\n{block}\n```", placeholder)
        placeholders.append((placeholder, block))

    # Convert markdown to HTML
    html_text = md.render(markdown_text)

    # Replace placeholders with rendered mermaid HTML
    for placeholder, mermaid_block in placeholders:
        mermaid_html = construct_mermaid_web(mermaid_block)
        html_text = html_text.replace(placeholder, mermaid_html)

    # Add basic HTML structure
    html_text = f"""
    <!DOCTYPE html>
    <html>
    <head>
        <meta charset="UTF-8">
        <meta name="viewport" content="width=device-width, initial-scale=1.0">
        <title>Egeria Report</title>
        <script src="https://cdn.jsdelivr.net/npm/mermaid@10/dist/mermaid.min.js"></script>
        <script>
            mermaid.initialize({{ startOnLoad: true }});
        </script>
        <style>
            body {{ font-family: Arial, sans-serif; line-height: 1.6; padding: 20px; }}
            h1 {{ color: #2c3e50; }}
            h2 {{ color: #3498db; }}
            pre {{ background-color: #f8f8f8; padding: 10px; border-radius: 5px; overflow-x: auto; }}
            table {{ border-collapse: collapse; width: 100%; margin-bottom: 20px; }}
            th, td {{ border: 1px solid #ddd; padding: 8px; text-align: left; }}
            th {{ background-color: #f2f2f2; }}
            tr:nth-child(even) {{ background-color: #f9f9f9; }}
        </style>
    </head>
    <body>
        {html_text}
    </body>
    </html>
    """

    return html_text

def make_preamble(obj_type: str, search_string: str, output_format: str = 'MD') -> Tuple[str, Optional[str]]:
    """
    Creates a preamble string and an elements action based on the given object type, search string,
    and output format.

    Args:
        obj_type: The type of object being updated or reported on (e.g., "Product", "Category").
        search_string: The search string used to filter objects. Defaults to "All Elements" if None.
        output_format: A format identifier determining the output structure.
            JSON - output standard json
            MD - output standard markdown with no preamble
            FORM - output markdown with a preamble for a form
            REPORT - output markdown with a preamble for a report

    Returns:
        tuple: A tuple containing:
            - A string representing the formatted update or report preamble.
            - A string or None indicating the action description for the elements,
              depending on the output format.
    """
    # search_string = search_string if search_string else "All Elements"
    elements_md = ""
    elements_action = "Update " + obj_type
    if output_format == "FORM":
        preamble = f"\n# Update {obj_type} Form - created at {datetime.now().strftime('%Y-%m-%d %H:%M')}\n"
        if search_string:
            preamble +=  f"\t {obj_type} found from the search string:  `{search_string}`\n\n"
        return preamble, elements_action
    elif output_format == "REPORT":
        elements_md += (f"# {obj_type} Report - created at {datetime.now().strftime('%Y-%m-%d %H:%M')}\n"
                       f"\t{obj_type}  found from the search string:  `{search_string}`\n\n")
        elements_action = None
        return elements_md, elements_action
    else:
        return "\n", elements_action

def materialize_egeria_summary(summary: dict, columns_struct: Optional[dict] = None) -> dict:
    """Recursively materialize an Egeria summary object into a clean dictionary.
    
    If columns_struct (a FormatSet dict) is provided, it uses the detail_spec definitions
    to promote nested elements of specific types into their own keys, matching the 
    report specification's expectation.
    """
    if not isinstance(summary, dict):
        return {}

    res = {}

    # 1. Header & properties materialization
    # Check if we are dealing with a summary that has a top-level elementHeader or properties
    # This handles both raw elements and summary objects
    header = summary.get("elementHeader") or (summary.get("relatedElement") or {}).get("elementHeader")
    props = summary.get("properties") or (summary.get("relatedElement") or {}).get("properties") or {}
    rel_props = summary.get("relationshipProperties") or {}

    # 1a. Generic materialization of the header
    if isinstance(header, dict):
        res["type"] = (header.get("type") or {}).get("typeName")
        res["guid"] = header.get("guid")
        
        # Pull everything else from the header generically
        for h_key, h_val in header.items():
            if h_key in ("guid", "type", "classifications", "origin", "versions", "status"):
                continue
            
            # Materialize this header property
            if isinstance(h_val, list):
                materialized_list = []
                for item in h_val:
                    if isinstance(item, dict):
                        # Recursive materialization
                        m_item = materialize_egeria_summary(item, columns_struct)
                        # Fallback for classifications/roles found in headers
                        if not m_item.get('name') and item.get('classificationName'):
                            m_item['name'] = item['classificationName']
                        materialized_list.append(m_item)
                    else:
                        materialized_list.append(item)
                res[h_key] = materialized_list
            elif isinstance(h_val, dict):
                res[h_key] = materialize_egeria_summary(h_val, columns_struct)
            else:
                res[h_key] = h_val

    # 1b. Materialize relationship properties
    if isinstance(rel_props, dict):
        for k, v in rel_props.items():
            if k not in ("class", "typeName", "extendedProperties") and k not in res:
                res[k] = v

    # 1c. Materialize element properties
    if isinstance(props, dict):
        # Add a friendly 'name' if possible
        if "displayName" in props:
            res["name"] = props["displayName"]
        elif "qualifiedName" in props:
            res["name"] = props["qualifiedName"]

        for k, v in props.items():
            if k not in ("class", "typeName", "extendedProperties"):
                if k not in res:
                    res[k] = v

    # 1d. Generic materialization of any other top-level fields
    for k, v in summary.items():
        if k in ("elementHeader", "properties", "relationshipProperties", "relatedElement", "nestedElements", "class"):
            continue
        if k in res:
            continue

        if isinstance(v, list):
            materialized_list = []
            for item in v:
                if isinstance(item, dict):
                    materialized_list.append(materialize_egeria_summary(item, columns_struct))
                else:
                    materialized_list.append(item)
            res[k] = materialized_list
        elif isinstance(v, dict):
            res[k] = materialize_egeria_summary(v, columns_struct)
        else:
            res[k] = v

    # 3. Build type-to-key map from the current spec if provided
    type_map = {}
    if columns_struct:
        formats = columns_struct.get('formats') or {}
        attributes = formats.get('attributes', [])
        for col in attributes:
            # col can be a dict or a Column object
            key = col.get('key') if isinstance(col, dict) else getattr(col, 'key', None)
            ds_name = col.get('detail_spec') if isinstance(col, dict) else getattr(col, 'detail_spec', None)
            
            if ds_name:
                # Resolve the spec to get its target_type with fallbacks (DICT -> LIST -> REPORT -> ALL)
                linked_spec = (
                    select_report_format(ds_name, "DICT")
                    or select_report_format(ds_name, "LIST")
                    or select_report_format(ds_name, "REPORT")
                    or select_report_format(ds_name, "ALL")
                )
                if linked_spec and linked_spec.get('target_type'):
                    type_map[linked_spec['target_type']] = key

    # 4. Process nested elements
    nested = summary.get("nestedElements") or []
    if isinstance(nested, list) and len(nested) > 0:
        if "nested_elements" not in res:
            res["nested_elements"] = []
            
        for n in nested:
            if not isinstance(n, dict):
                continue
            # Descend into next level. 
            extracted = materialize_egeria_summary(n, columns_struct)
            res["nested_elements"].append(extracted)

            # Generic promotion: If the type of this nested element matches a column in our spec,
            # add it to that specific key so populate_columns_from_properties finds it.
            element_type = extracted.get("type")
            if element_type in type_map:
                target_key = type_map[element_type]
                if target_key not in res:
                    res[target_key] = []
                res[target_key].append(extracted)

    return res

def render_rich_value(value: Any, output_format: str) -> str:
    """Visually format rich nested values for text-based reports.
    - LIST: summarized string (handled elsewhere)
    - FORM: keep compact summary to preserve editability
    - REPORT/MD: nested bullet lists
    """
    def summarize(v: Any) -> str:
        if isinstance(v, dict):
            return v.get('name') or v.get('displayName') or v.get('qualifiedName') or v.get('type') or str(v)
        if isinstance(v, list):
            names = [summarize(i) for i in v]
            return ", ".join([x for x in names if x])
        return str(v)

    if output_format == 'FORM':
        return summarize(value)

    # REPORT/MD nested bullets
    def bullets(v: Any, indent: int = 0) -> str:
        if not isinstance(v, (list, dict)):
            return str(v)
        prefix = "  " * indent
        out = []
        if isinstance(v, list):
            for item in v:
                if isinstance(item, dict):
                    name = item.get('name') or item.get('type', 'Item')
                    out.append(f"{prefix}* **{name}**")
                    for k, val in item.items():
                        if k in ('name', 'nested_elements'): continue
                        out.append(f"{prefix}  * {camel_to_title_case(k)}: {val}")
                    if item.get('nested_elements'):
                        out.append(bullets(item['nested_elements'], indent + 2))
                else:
                    out.append(f"{prefix}* {item}")
        else:  # dict
            for k, val in v.items():
                if k == 'nested_elements':
                    out.append(bullets(val, indent + 1))
                else:
                    out.append(f"{prefix}* **{camel_to_title_case(k)}**: {val}")
        return "\n".join(out)

    return bullets(value, 0)

def make_md_attribute(attribute_name: str, attribute_value: str|list|dict, output_type: str) -> Optional[str]:
    """
    Create a markdown attribute line for a given attribute name and value.

    Args:
        attribute_name: The name of the attribute
        attribute_value: The value of the attribute
        output_type: The output format (FORM, MD, REPORT)

    Returns:
        str: Formatted markdown for the attribute
    """
    output = ""
    if isinstance(attribute_value,str):
        attribute_value = attribute_value.strip() if attribute_value else ""
    
    if attribute_name:
        attribute_title = camel_to_title_case(attribute_name)
    else:
        attribute_title = ""

    if isinstance(attribute_value, (dict, list)):
        rendered = render_rich_value(attribute_value, output_type)
        if rendered:
            return f"## {attribute_title}\n{rendered}\n\n"
        else:
            return "\n"

    if output_type in ["FORM", "MD"]:
        if attribute_name.lower() in [ "mermaid", "solutionBlueprintMermaidGraph", "links", "implemented by", "sub_components"]:
            return '\n'

        output = f"## {attribute_title}\n{attribute_value}\n\n"
    elif output_type in ["REPORT", "MERMAID"]:
        if attribute_title in MERMAID_GRAPH_TITLES + ["Mermaid Graph", "Mermaid"]:
            norm_mermaid = _normalize_mermaid_graph(attribute_value)
            output = f"## {attribute_title}\n\n```mermaid\n{norm_mermaid}\n```\n"
        elif attribute_value:
            output = f"## {attribute_title}\n{attribute_value}\n\n"
    return output

def format_for_markdown_table(text: str, guid: str = None) -> str:
    """
    Format text for markdown tables by replacing newlines with spaces and escaping pipe characters.
    No truncation is applied to allow full-length text display regardless of console width.

    Args:
        text (str): The text to format

    Returns:
        str: Formatted text safe for markdown tables
    """
    if text is None:
        return ""
    if isinstance(text, list):
        text = "\n".join([str(i) for i in text])
    if not isinstance(text, str):
        text = str(text)

    t = text.replace("\n", " ").replace("|", "\\|")
    if '::' in t and guid:
        t = f" [{t}](#{guid}) "
    return t


def populate_columns_from_properties(element: dict, columns_struct: dict) -> dict:
    """
    Populate a columns_struct with values from the element's properties.

    The element can be either:
      - a full element dict that contains a nested 'properties' dict (preferred), or
      - a dict that itself represents the set of properties.

    In both cases, the effective properties are expected to use camelCase keys.
    The columns_struct is expected to follow the format returned by select_report_spec, where
    columns are located at columns_struct['formats']['columns'] and each column is a dict or Column object
    containing at least a 'key' field. For each column whose key corresponds
    to a key in the element properties (either directly or after converting to camelCase),
    this function adds a 'value' entry to the column with the matching property's value.

    Args:
        element: Either a dict with a nested 'properties' dict using camelCase keys, or
                 a dict that itself is the set of properties.
        columns_struct: The columns structure whose columns have 'key' fields.

    Returns:
        The updated columns_struct (the input structure is modified in place and also returned).
    """
    if not isinstance(columns_struct, dict):
        return columns_struct

    # Determine the effective properties:
    # 1) Prefer element['properties'] if present and a dict
    # 2) Otherwise, if element itself is a dict, treat it as the properties dict
    # 3) Otherwise, nothing to do
    props: dict | None = None
    if isinstance(element, dict):
        maybe_props = element.get('properties')
        if not isinstance(maybe_props, dict):
            # NEW: Check for relatedElement properties (common in relationship results)
            related = element.get('relatedElement') or element.get('related_element')
            if isinstance(related, dict):
                maybe_props = related.get('properties')

        props = maybe_props if isinstance(maybe_props, dict) else element
    if not isinstance(props, dict):
        return columns_struct

    # Get the attributes list if present
    formats = columns_struct.get('formats') or {}
    columns = formats.get('attributes') if isinstance(formats, dict) else None
    if not isinstance(columns, list):
        return columns_struct

    # Pre-compute snake_case to camelCase mapping for all columns
    key_mapping = {}
    for col in columns:
        key_snake = col.get('key') if isinstance(col, dict) else getattr(col, 'key', None)
        if key_snake:
            key_mapping[key_snake] = to_camel_case(key_snake)

    # Single pass to populate values
    for col in columns:
        try:
            key_snake = col.get('key') if isinstance(col, dict) else getattr(col, 'key', None)
            if not key_snake:
                continue
            
            # Helper to set value in col
            def set_col_value(val):
                if isinstance(col, dict):
                    col['value'] = val
                else:
                    setattr(col, 'value', val)

            # 1) Try original key (handles already camelCased keys like assignmentType)
            val = props.get(key_snake)
            # Fallback to root element if not found in nested properties
            if val is None and props is not element:
                val = element.get(key_snake)

            if val is not None:
                set_col_value(val)
                continue

            # 2) Try pre-computed camelCase key
            key_camel = key_mapping.get(key_snake)
            if key_camel:
                val = props.get(key_camel)
                # Fallback to root element if not found in nested properties
                if val is None and props is not element:
                    val = element.get(key_camel)
                
                if val is not None:
                    set_col_value(val)
                    continue
            
            # 3) Try uppercase (handles GUID if it was extracted as GUID)
            key_upper = key_snake.upper()
            val = props.get(key_upper)
            # Fallback to root element if not found in nested properties
            if val is None and props is not element:
                val = element.get(key_upper)
            
            if val is not None:
                set_col_value(val)
                continue

        except Exception as e:
            # Be resilient; log and continue
            logger.debug(f"populate_columns_from_properties: skipping column due to error: {e}")
            continue

    return columns_struct

def populate_feedback(element: dict, columns_struct: dict) -> dict:
    """
    Populate a columns_struct with values from the element's feedback.

    The element dict may have feedback relationships. If these are present in the column request, extract them
    and add the values to the columns_struct.
    The columns_struct is expected to follow the format returned by select_report_spec, where
    columns are located at columns_struct['formats']['columns'] and each column is a dict containing
    at least a 'key' field expressed in snake_case. For each column whose snake_case key corresponds
    to a key in the element properties (after converting to camelCase), this function adds a 'value'
    entry to the column with the matching property's value.

    Args:
        element: The element containing a 'properties' dict with camelCase keys.
        columns_struct: The columns structure whose columns have snake_case 'key' fields.

    Returns:
        The updated columns_struct (the input structure is modified in place and also returned).
    """
    if not isinstance(columns_struct, dict):
        return columns_struct

    # Need to think more about what I want to produce
    return columns_struct
    props = {}

    # Get the attributes list if present
    formats = columns_struct.get('formats') or {}
    columns = formats.get('attributes') if isinstance(formats, dict) else None
    if not isinstance(columns, list):
        return columns_struct

    for col in columns:
        try:
            key_snake = col.get('key') if isinstance(col, dict) else None
            if not key_snake:
                continue
            # Convert the snake_case key to camelCase to look up in properties
            key_camel = to_camel_case(key_snake)
            if key_camel in props:
                col['value'] = props.get(key_camel)
        except Exception as e:
            # Be resilient; log and continue
            logger.debug(f"populate_columns_from_properties: skipping column due to error: {e}")
            continue

    return columns_struct

def get_required_relationships(element: dict, columns_struct: dict) -> dict:
    """
    Populate relationship-derived column values in columns_struct based on top-level keys in the element.
    
    NOTE: This function is now a lightweight wrapper around the optimized logic in populate_common_columns.
    For best performance, use populate_common_columns directly with include_relationships=True.

    This function inspects the requested columns in columns_struct, converts each column key from
    snake_case to camelCase, and if a matching top-level key exists in the element, parses that value
    (typically lists of relationship beans) into a human-readable value (e.g., a comma-separated list
    of qualified names) and stores it under the column's 'value'. Columns not specified in the
    columns_struct are ignored. Existing non-empty 'value's are left as-is.

    Example: if a column with key 'member_of_collections' is present, this function will look for the
    top-level key 'memberOfCollections' in the element and derive a value if found.

    Args:
        element: The element dictionary containing top-level relationship lists (e.g., associatedGlossaries,
                 memberOfCollections, collectionMembers).
        columns_struct: The columns structure to augment with derived 'value's.

    Returns:
        The updated columns_struct (modified in place and returned).
    """
    if not isinstance(columns_struct, dict):
        return columns_struct

    formats = columns_struct.get('formats') or {}
    columns = formats.get('attributes') if isinstance(formats, dict) else None
    if not isinstance(columns, list):
        return columns_struct

    # Pre-compute relationship values for efficiency
    relationship_values = {}
    for col in columns:
        if not isinstance(col, dict):
            continue
        key_snake = col.get('key')
        if not key_snake:
            continue
        # If already has a non-empty value, don't overwrite
        if col.get('value') not in (None, ""):
            continue
            
        key_camel = to_camel_case(key_snake)
        if key_camel not in element:
            continue

        top_val = element.get(key_camel)
        derived_value = ""
        if isinstance(top_val, list):
            names = []
            for item in top_val:
                nm = _extract_name_from_relationship_item(item)
                if nm:
                    names.append(nm)
            derived_value = ", ".join(names)
        elif isinstance(top_val, dict):
            nm = _extract_name_from_relationship_item(top_val)
            derived_value = nm or ""
        else:
            derived_value = str(top_val) if top_val is not None else ""
        
        if derived_value:
            relationship_values[key_snake] = derived_value

    # Apply the values
    for col in columns:
        if isinstance(col, dict):
            key = col.get('key')
            if key and key in relationship_values and col.get('value') in (None, ""):
                col['value'] = relationship_values[key]

    return columns_struct




def generate_entity_md(elements: List[Dict], 
                      elements_action: str, 
                      output_format: str, 
                      entity_type: str,
                      extract_properties_func: Callable, 
                      get_additional_props_func: Optional[Callable] = None,
                       columns_struct: [dict] = None) -> str:
    """
    Generic method to generate markdown for entities.

    Args:
        elements (list): List of entity elements
        elements_action (str): Action description for elements
        output_format (str): Output format
        entity_type (str): Type of entity (Glossary, Term, Category, etc.)
        extract_properties_func: Function to extract properties from an element
        get_additional_props_func: Optional function to get additional properties
        columns (list): List of column name structures

    Returns:
        str: Markdown representation
    """
    heading = columns_struct.get("heading")
    if heading == "Default Base Attributes":
        elements_md = "## Reporting on Default Base Attributes - Perhaps couldn't find a valid combination of report_spec and output_format?\n\n"
    else:
        elements_md = ""
    base_columns = columns_struct['formats'].get('attributes') if columns_struct else None

    for element in elements:
        if element is None:
                continue
        guid = element.get('elementHeader', {}).get('guid')

        # Prefer new behavior: extractor returns an updated columns_struct with values
        returned_struct = None
        if columns_struct is not None:
            try:
                returned_struct = extract_properties_func(element, columns_struct)
            except TypeError:
                # Fallback for legacy extractors without columns_struct parameter
                returned_struct = None
        
        # Legacy fallback: get props dict if no columns_struct provided/returned
        props = {}
        if returned_struct is None:
            props = extract_properties_func(element) if callable(extract_properties_func) else {}

        # Get additional properties if function is provided
        additional_props = {}
        if get_additional_props_func:
            # Use guid if available, else try to get from props
            guid_for_fmt = guid or props.get('GUID')
            additional_props = get_additional_props_func(element, guid_for_fmt, output_format)

        # Determine display name
        display_name = None
        if returned_struct is not None:
            cols = returned_struct.get('formats', {}).get('attributes', [])
            # Find value from 'display_name' or 'title'
            for col in cols:
                if col.get('key') in ('display_name', 'title', 'keyword', 'full_name'):
                    display_name = col.get('value')
                    if display_name:
                        break
        else:
            display_name = props.get('display_name') or props.get('title') or props.get('keyword') or props.get('full_name') or props.get('fullName')

        if display_name is None and (keyword:= element.get('properties',{}).get('keyword',None)) is not None:
            display_name = keyword
        elif display_name is None:
            display_name = "NO DISPLAY NAME"

        # Format header based on output format
        if output_format in ['FORM', 'MD']:
            elements_md += f"# {elements_action}\n\n"
            elements_md += f"## {entity_type} Name \n\n{display_name}\n\n"
        elif output_format == 'REPORT':
            elements_md += f'<a id="{(guid or props.get("GUID") or "No GUID" )}"></a>\n# {entity_type} Name: {display_name}\n\n'
        else:
            elements_md += f"## {entity_type} Name \n\n{display_name}\n\n"

        # Add attributes based on column spec if available, otherwise, add all (legacy)
        if returned_struct is not None:
            cols = returned_struct.get('formats', {}).get('attributes', [])
            for column in cols:
                name = column.get('name')
                key = column.get('key')
                value = column.get('value')
                detail_spec = column.get('detail_spec')

                if value in (None, "") and key in additional_props:
                    value = additional_props[key]
                if column.get('format'):
                    value = format_for_markdown_table(value, guid)
                
                elements_md += make_md_attribute(name, value, output_format)

                # Master-Detail support for REPORT, MD, and FORM formats
                if detail_spec and value and output_format in ['REPORT', 'MD', 'FORM']:
                    values_list = value if isinstance(value, list) else [value]
                    if values_list and all(isinstance(v, dict) for v in values_list):
                        # Resolve the linked spec
                        detail_struct = select_report_format(detail_spec, 'REPORT')
                        if detail_struct:
                            elements_md += f"\n### {name} Details\n\n"
                            elements_md += generate_output(
                                elements=values_list,
                                search_string="",
                                entity_type=detail_struct.get('target_type') or name,
                                output_format='REPORT',
                                extract_properties_func=populate_columns_from_properties,
                                columns_struct=detail_struct,
                                include_preamble=False,
                            )
                            elements_md += "\n"
            if wk := returned_struct.get("annotations", {}).get("wikilinks", None):
                elements_md += ", ".join(wk)
        elif base_columns:
            # If we have columns but extractor didn't return struct, use legacy props lookup
            for column in base_columns:
                key = column['key']
                name = column['name']
                value = ""
                if key in props:
                    value = props[key]
                elif key in additional_props:
                    value = additional_props[key]
                if column.get('format'):
                    value = format_for_markdown_table(value, guid or props.get('GUID'))
                elements_md += make_md_attribute(name, value, output_format)
            if wk := columns_struct.get("annotations", {}).get("wikilinks", None):
                elements_md += ", ".join(wk)
        else:
            # Legacy path without columns: dump all props
            for key, value in props.items():
                if output_format in ['FORM', 'MD', 'DICT'] and key == 'mermaid':
                    continue
                if key not in ['properties', 'display_name']:
                    if key == 'mermaid' and value == '':
                        continue
                    elements_md += make_md_attribute(key.replace('_', ' '), value, output_format)
            for key, value in additional_props.items():
                elements_md += make_md_attribute(key.replace('_', ' '), value, output_format)

        if element != elements[-1]:
            elements_md += MD_SEPARATOR

    return elements_md

def generate_entity_md_table(elements: List[Dict],
                            search_string: str,
                            entity_type: str,
                            extract_properties_func: Callable,
                            columns_struct: dict,
                            get_additional_props_func: Optional[Callable] = None,
                            output_format: str = 'LIST') -> str:
    """
    Generic method to generate a markdown table for entities.

    Args:
        elements (list): List of entity elements
        search_string (str): The search string used
        entity_type (str): Type of entity (Glossary, Term, Category, etc.)
        extract_properties_func: Function to extract properties from an element
        columns: List of column definitions, each containing 'name', 'key', and 'format' (optional)
        get_additional_props_func: Optional function to get additional properties
        output_format (str): Output format (FORM, REPORT, LIST, etc.)

    Returns:
        str: Markdown table
    """
    # Handle pluralization - if entity_type ends with 'y', use 'ies' instead of 's'
    target_type = columns_struct.get('target_type', entity_type)
    # if target_type.endswith('y'):
    #     target_type = target_type.replace('y', 'ies')
    # else:
    #     target_type = target_type.replace('s', 's')

    entity_type_plural = f"{target_type[:-1]}ies" if target_type.endswith('y') else f"{target_type}s"
    # entity_type_plural = target_type
    columns = columns_struct['formats'].get('attributes', [])
    heading = columns_struct.get("heading")
    if heading == "Default Base Attributes":
        elements_md = "## Reporting on Default Base Attributes - Perhaps couldn't find a valid combination of report_spec and output_format?\n\n"
    else:
        elements_md = ""

    if output_format == "LIST":
        elements_md = f"# {entity_type_plural} Table\n\n"
        elements_md += f"{entity_type_plural} found from the search string: `{search_string}`\n\n"

    # Add column headers
    header_row = "| "
    separator_row = "|"
    for column in columns:
        header_row += f"{column['name']} | "
        separator_row += "-------------|"

    elements_md += header_row + "\n"
    elements_md += separator_row + "\n"

    details_md = ""

    for element in elements:
        guid = element.get('elementHeader', {}).get('guid', None)

        # Extractor returns columns_struct with values when possible
        # Use shallow copy and reset only column values for performance
        local_columns_struct = columns_struct.copy()
        if 'formats' in local_columns_struct and 'attributes' in local_columns_struct['formats']:
            # Create new list of columns with reset values
            local_columns_struct['formats'] = local_columns_struct['formats'].copy()
            local_columns_struct['formats']['attributes'] = [
                {**col, 'value': None} if isinstance(col, dict) else col
                for col in local_columns_struct['formats']['attributes']
            ]
        try:
            returned_struct = extract_properties_func(element, local_columns_struct)
        except TypeError:
            returned_struct = None

        # For help mode, bypass extraction
        if output_format == "help":
            returned_struct = {"formats": {"attributes": columns}}

        # Additional props (if any)
        additional_props = {}
        if get_additional_props_func:
            additional_props = get_additional_props_func(element, guid, output_format)

        # Heuristic to get display name for section headings
        display_name_for_row = None
        if returned_struct is not None:
            for col in returned_struct.get('formats', {}).get('attributes', []):
                if col.get('key') in ('display_name', 'title', 'keyword', 'full_name') and col.get('value'):
                    display_name_for_row = col['value']
                    break

        # Build row
        row = "| "
        pending_details = []  # (anchor_id, col_display_name, detail_spec, values)

        if returned_struct is not None:
            for column in returned_struct.get('formats', {}).get('attributes', []):
                key = column.get('key')
                value = column.get('value')
                if (value in (None, "")) and key in additional_props:
                    value = additional_props[key]

                # Summarize for LIST
                cell_value = value
                if isinstance(value, list):
                    names = []
                    for item in value:
                        if isinstance(item, dict):
                            nm = item.get('name') or item.get('displayName') or item.get('qualifiedName') or item.get('type')
                            if nm: names.append(nm)
                        elif item is not None:
                            names.append(str(item))
                    cell_value = ", ".join(names)
                elif isinstance(value, dict):
                    cell_value = value.get('name') or value.get('displayName') or value.get('qualifiedName') or str(value)

                # If detail_spec present and we have values to show later, add a link
                detail_spec = column.get('detail_spec')
                if detail_spec and value not in (None, "", [], {}):
                    anchor_id = f"{guid or 'noguid'}_{key}"
                    # show compact + link
                    summary_label = f"{len(value)} items" if isinstance(value, list) else (cell_value or "detail")
                    link = f" [details](#{anchor_id})"
                    cell_value = (cell_value + link) if cell_value else (summary_label + link)

                    # queue rendering of the detail section (values are already materialized dicts)
                    values_list = value if isinstance(value, list) else [value]
                    pending_details.append((anchor_id, column.get('name', key), detail_spec, values_list))

                if value in (None, "", [], {}):
                    cell_value = " --- "

                if column.get('format'):
                    cell_value = format_for_markdown_table(cell_value, guid)
                elif isinstance(cell_value, str):
                    cell_value = cell_value.replace("\n", " ").replace("|", "\\|")

                row += f"{cell_value} | "
        else:
            # Legacy fallback: read from props dict
            props = extract_properties_func(element)
            for column in columns:
                key = column['key']
                value = " "
                if key in props:
                    value = props[key]
                elif key in additional_props:
                    value = additional_props[key]

                if value in (None, "", " "):
                    value = " --- "

                if column.get('format'):
                    value = format_for_markdown_table(value, guid or props.get('GUID'))
                elif isinstance(value, str):
                    value = value.replace("\n", " ").replace("|", "\\|")

                row += f"{value} | "

        elements_md += row + "\n"

        # After the row, append queued detail sections
        for anchor_id, col_disp, spec_name, values_list in pending_details:
            # Resolve the linked spec in an appropriate format (prefer LIST for compactness)
            detail_struct = resolve_output_formats(entity_type, 'LIST', spec_name, default_label=spec_name)
            if detail_struct is None:
                # fallback to REPORT if LIST not available
                detail_struct = resolve_output_formats(entity_type, 'REPORT', spec_name, default_label=spec_name)

            # Render by reusing the generic pipeline; treat each dict as a properties payload
            # so populate_columns_from_properties can do key-mapping
            if detail_struct is not None:
                details_md += f"\n\n<a id=\"{anchor_id}\"></a>\n### {col_disp} for {display_name_for_row or (guid or '')}\n\n"
                details_md += generate_output(
                    elements=values_list,               # list of dicts already materialized
                    search_string="",
                    entity_type=detail_struct.get('target_type') or col_disp,
                    output_format='LIST' if 'LIST' in detail_struct.get('formats', {}).get('types', ['LIST']) else 'REPORT',
                    extract_properties_func=populate_columns_from_properties,
                    columns_struct=detail_struct,
                    include_preamble=False,
                )

    elements_md += details_md
    return elements_md


def generate_entity_dict(elements: List[Dict], 
                        extract_properties_func: Callable, 
                        get_additional_props_func: Optional[Callable] = None,
                        include_keys: Optional[List[str]] = None, 
                        exclude_keys: Optional[List[str]] = None,
                        columns_struct: dict = None,
                        output_format: str = 'DICT') -> List[Dict]:
    """
    Generic method to generate a dictionary representation of entities.

    Args:
        elements (list): List of entity elements
        extract_properties_func: Function to extract properties from an element
        get_additional_props_func: Optional function to get additional properties
        include_keys: Optional list of keys to include in the result (if None, include all)
        exclude_keys: Optional list of keys to exclude from the result (if None, exclude none)
        columns_struct: Optional dict of columns to include (if None, include all)
        output_format (str): Output format (FORM, REPORT, DICT, etc.)

    Returns:
        list: List of entity dictionaries
    """
    result = []

    #####
    # Add attributes based on column spec if available, otherwise, add all
    import copy
    for element in elements:
        if not isinstance(element, dict):
            continue

        guid = element.get('elementHeader', {}).get('guid')

        # Use shallow copy and reset only column values for performance
        local_columns_struct = None
        if columns_struct is not None:
            local_columns_struct = columns_struct.copy()
            if 'formats' in local_columns_struct and 'attributes' in local_columns_struct['formats']:
                # Create new list of columns with reset values
                local_columns_struct['formats'] = local_columns_struct['formats'].copy()
                local_columns_struct['formats']['attributes'] = [
                    {**col, 'value': None} if isinstance(col, dict) else col
                    for col in local_columns_struct['formats']['attributes']
                ]

        returned_struct = None
        if local_columns_struct is not None:
            try:
                returned_struct = extract_properties_func(element, local_columns_struct)
            except TypeError as e:
                logger.info(f"Error - didn't find extractor?: {e}")
                returned_struct = None

        # Get additional properties if function is provided
        additional_props = {}
        if get_additional_props_func:
            additional_props = get_additional_props_func(element, guid, output_format)

        # Create entity dictionary
        entity_dict = {}

        cols = local_columns_struct['formats'].get('attributes', None) if local_columns_struct else None
        if returned_struct is not None:
            for column in returned_struct.get('formats', {}).get('attributes', []):
                key = column.get('key')
                name = column.get('name')
                value = column.get('value')
                detail_spec = column.get('detail_spec')
                
                if (value in (None, "")) and key in additional_props:
                    value = additional_props[key]
                if column.get('format'):
                    value = format_for_markdown_table(value, guid)

                # Master-Detail support for DICT format
                if detail_spec and value and output_format in ['DICT', 'TABLE']:
                    values_list = value if isinstance(value, list) else [value]
                    if values_list and all(isinstance(v, dict) for v in values_list):
                        # Use DICT for nested details if top-level is TABLE
                        recursive_ofmt = 'DICT' if output_format == 'TABLE' else output_format
                        detail_struct = select_report_format(detail_spec, recursive_ofmt)
                        if detail_struct:
                            value = generate_output(
                                elements=values_list,
                                search_string="",
                                entity_type=detail_struct.get('target_type') or name,
                                output_format=recursive_ofmt,
                                extract_properties_func=populate_columns_from_properties,
                                columns_struct=detail_struct,
                                include_preamble=False,
                            )

                # Avoid overwriting when multiple columns share the same display name in a spec
                dict_key = name
                if dict_key in entity_dict:
                    logger.warning(f"DICT key collision for display name '{dict_key}'. Suffixing duplicate to preserve all values.")
                    suffix_idx = 1
                    tmp_key = f"{dict_key}_{suffix_idx}"
                    while tmp_key in entity_dict:
                        suffix_idx += 1
                        tmp_key = f"{dict_key}_{suffix_idx}"
                    dict_key = tmp_key
                entity_dict[dict_key] = value
        elif cols:
            for column in cols:
                key = column['key']
                name = column['name']
                value = ""
                props = extract_properties_func(element, columns_struct)
                if key in props:
                    value = props[key]
                elif key in additional_props:
                    value = additional_props[key]
                if  column.get('format', None):
                    value = format_for_markdown_table(value, guid or props.get('GUID'))
                dict_key = name
                if dict_key in entity_dict:
                    logger.warning(f"DICT key collision for display name '{dict_key}'. Suffixing duplicate to preserve all values.")
                    suffix_idx = 1
                    tmp_key = f"{dict_key}_{suffix_idx}"
                    while tmp_key in entity_dict:
                        suffix_idx += 1
                        tmp_key = f"{dict_key}_{suffix_idx}"
                    dict_key = tmp_key
                entity_dict[dict_key] = value
        else:
            props = extract_properties_func(element, local_columns_struct)
            # Add properties based on include/exclude lists
            for key, value in props.items():
                if key not in ['properties', 'mermaid']:  # Skip the raw properties object
                    if (include_keys is None or key in include_keys) and (
                            exclude_keys is None or key not in exclude_keys):
                        entity_dict[key] = value

            # Add additional properties
            for key, value in additional_props.items():
                if (include_keys is None or key in include_keys) and (exclude_keys is None or key not in exclude_keys):
                    entity_dict[key] = value

        result.append(entity_dict)
    #####
    # for element in elements:
    #     if element is None:
    #         continue
    #     props = extract_properties_func(element)
    #
    #     # Get additional properties if function is provided
    #     additional_props = {}
    #     if get_additional_props_func:
    #         additional_props = get_additional_props_func(element,props['GUID'], output_format)
    #
    #     # Create entity dictionary
    #     entity_dict = {}
    #
    #     # Add properties based on include/exclude lists
    #     for key, value in props.items():
    #         if key not in [ 'properties', 'mermaid']:  # Skip the raw properties object
    #             if (include_keys is None or key in include_keys) and (
    #                     exclude_keys is None or key not in exclude_keys):
    #                 entity_dict[key] = value
    #
    #     # Add additional properties
    #     for key, value in additional_props.items():
    #         if (include_keys is None or key in include_keys) and (exclude_keys is None or key not in exclude_keys):
    #             entity_dict[key] = value
    #
    #     result.append(entity_dict)

    return result

def resolve_output_formats(entity_type: str,
                           output_format: str,
                           report_spec: Optional[Union[str, dict]] = None,
                           default_label: Optional[str] = None,
                           **kwargs) -> Optional[dict]:
    """
    Resolve a report format structure given an entity type, the desired output format
    (e.g., DICT, LIST, MD, REPORT, FORM), and either a label (str) or a dict of format sets.

    Notes:
    - Preferred naming is report_spec (formerly output_format); compatibility for the
      old 'output_format_spec' terminology has been removed.

    Selection order:
    - If report_spec is a str: select by label.
    - If report_spec is a dict: use get_report_spec_match to pick a matching format.
    - Else: try selecting by entity_type or default_label.
    - Fallback: select "Default".
    """
    from pyegeria.view.base_report_formats import get_report_spec_match

    if report_spec is None and isinstance(kwargs, dict):
        if 'report_spec' in kwargs:
            report_spec = kwargs.get('report_spec')
        elif 'report_format' in kwargs:
            # Still accept 'report_format' as a synonym for report_spec
            report_spec = kwargs.get('report_format')

    if isinstance(report_spec, str):
        return select_report_format(report_spec, output_format)
    if isinstance(report_spec, dict):
        return get_report_spec_match(report_spec, output_format)

    label = default_label or entity_type
    fmt = select_report_format(label, output_format)
    if fmt is None:
        fmt = select_report_format("Default", output_format)
    return fmt


def overlay_additional_values(columns_struct: dict, extra: Optional[dict]) -> dict:
    """
    Overlay extra values into columns_struct only where the column's value is empty or missing.
    Returns the modified columns_struct.
    """
    if not isinstance(columns_struct, dict) or not extra:
        return columns_struct
    columns = columns_struct.get('formats', {}).get('attributes')
    if not isinstance(columns, list):
        return columns_struct
    for col in columns:
        if not isinstance(col, dict):
            continue
        key = col.get('key')
        if not key:
            continue
        if col.get('value') in (None, "") and key in extra:
            col['value'] = extra[key]
    return columns_struct


def populate_common_columns(
    element: dict,
    columns_struct: dict,
    *,
    include_header: bool = True,
    include_relationships: bool = True,
    include_subject_area: bool = True,
    mermaid_source_key: str = 'mermaidGraph',
    mermaid_dest_key: str = 'mermaid'
) -> dict:
    """
    Populate the common columns in columns_struct based on a standard Egeria element shape.

    Steps:
    - Populate from element.properties (camelCase mapped from snake_case keys)
    - Optionally overlay header-derived values (GUID, type_name, times, etc.)
    - Optionally populate relationship-based columns via get_required_relationships
    - Optionally populate subject_area from element.elementHeader.subjectArea.classificationProperties.subjectAreaName
    - If a column with key == mermaid_dest_key is present, set it from mermaid_source_key
    - Do not overwrite non-empty values already set
    """
    # Pre-compute values that will be reused
    props = element.get('properties', {}) if isinstance(element, dict) else {}
    header_props = _extract_referenceable_properties(element) if include_header else {}
    guid = header_props.get('GUID') if include_header else None
    
    # Pre-compute subject area value
    subject_area_val = ""
    if include_subject_area:
        try:
            subject_area = element.get('elementHeader', {}).get('subjectArea') or ""
            if isinstance(subject_area, dict):
                subject_area_val = subject_area.get('classificationProperties', {}).get('subjectAreaName', '')
        except Exception as e:
            logger.debug(f"populate_common_columns: subject_area handling error: {e}")
    
    # Pre-compute mermaid value
    mermaid_val = element.get(mermaid_source_key, '') or ''
    
    # Pre-compute relationship values for efficiency
    relationship_values = {}
    if include_relationships:
        formats = columns_struct.get('formats') or {}
        columns = formats.get('attributes') if isinstance(formats, dict) else None
        if isinstance(columns, list):
            for col in columns:
                if not isinstance(col, dict):
                    continue
                key_snake = col.get('key')
                if not key_snake:
                    continue
                key_camel = to_camel_case(key_snake)
                if key_camel in element:
                    top_val = element.get(key_camel)
                    derived_value = ""
                    if isinstance(top_val, list):
                        if col.get('detail_spec'):
                            relationship_values[key_snake] = top_val
                            continue
                        names = []
                        for item in top_val:
                            nm = _extract_name_from_relationship_item(item)
                            if nm:
                                names.append(nm)
                        derived_value = ", ".join(names)
                    elif isinstance(top_val, dict):
                        if col.get('detail_spec'):
                            relationship_values[key_snake] = top_val  # Preserve raw dict for master-detail
                            continue
                        nm = _extract_name_from_relationship_item(top_val)
                        derived_value = nm or ""
                    else:
                        derived_value = str(top_val) if top_val is not None else ""
                    if derived_value:
                        relationship_values[key_snake] = derived_value
    
    # Single pass through columns - consolidate all operations
    col_data = columns_struct
    columns_list = col_data.get('formats', {}).get('attributes', [])
    header = element.get('elementHeader', {}) if isinstance(element, dict) else {}
    
    for column in columns_list:
        if not isinstance(column, dict):
            continue
        
        key = column.get('key')
        if not key:
            continue
        
        # Skip if already has a value
        if column.get('value') not in (None, ""):
            continue
        
        # 1) Try properties (camelCase conversion)
        key_camel = to_camel_case(key)
        if key_camel in props:
            column['value'] = props.get(key_camel)
            continue
        
        # 1a) Try elementHeader (generic fallback)
        if key_camel in header:
            h_val = header.get(key_camel)
            if isinstance(h_val, list):
                # Summarize list of objects (like projectRoles)
                names = []
                for item in h_val:
                    if isinstance(item, dict):
                        # Try to find a name generically
                        nm = item.get('classificationName') or item.get('name') or item.get('displayName') or item.get('qualifiedName')
                        if nm:
                            names.append(nm)
                        else:
                            # If no name found, maybe it's a simple dict we can stringify or just skip
                            names.append(str(item))
                    else:
                        names.append(str(item))
                column['value'] = ", ".join(names)
            elif isinstance(h_val, dict):
                # Generic fallback for dicts in header
                column['value'] = h_val.get('name') or h_val.get('displayName') or h_val.get('qualifiedName') or str(h_val)
            else:
                column['value'] = h_val
            continue

        # 2) Try header properties (from pre-extracted referenceable properties)
        if include_header:
            if key in header_props:
                column['value'] = header_props.get(key)
                continue
            elif isinstance(key, str) and key.lower() == 'guid':
                column['value'] = guid
                continue
        
        # 3) Try relationship values
        if include_relationships and key in relationship_values:
            column['value'] = relationship_values[key]
            continue
        
        # 4) Try subject area
        if include_subject_area and key == 'subject_area':
            column['value'] = subject_area_val
            continue
        
        # 5) Try mermaid
        if key == mermaid_dest_key:
            column['value'] = mermaid_val
            continue
        if key in MERMAID_GRAPHS and mermaid_dest_key not in MERMAID_GRAPHS:
            column['value'] = element.get(key, '')
            continue
    
    # Get any Feedback Values that have been requested (kept separate as it may have side effects)
    col_data = populate_feedback(element, col_data)
    
    return col_data


def _extract_name_from_relationship_item(item: Any) -> Optional[str]:
    """Best-effort extraction of a display/qualified name from a relationship item."""
    try:
        if isinstance(item, dict):
            # Common pattern: item['relatedElement']['properties']['qualifiedName']
            related = item.get('relatedElement') or item.get('related_element')
            if isinstance(related, dict):
                props = related.get('properties') or {}
                name = (
                    props.get('qualifiedName')
                    or props.get('displayName')
                    or props.get('name')
                )
                if name:
                    return name
            # Sometimes the properties may be at the top level of the item
            name = (
                item.get('qualifiedName')
                or item.get('displayName')
                or item.get('name')
            )
            if name:
                return name
        elif isinstance(item, str):
            return item
    except Exception as e:
        logger.debug(f"_extract_name_from_relationship_item: error extracting name: {e}")
    return None


def extract_mermaid_only(elements: Union[Dict, List[Dict]]) -> Union[str, List[str]]:
    """
    Extract mermaid graph data from elements.

    Args:
        elements: Dictionary or list of dictionaries containing element data

    Returns:
        String or list of strings containing mermaid graph data
    """
    if isinstance(elements, dict):
        mer = elements.get("mermaidGraph", None)
        if mer:
            norm_mermaid = _normalize_mermaid_graph(mer)
            return f"\n```mermaid\n{norm_mermaid}\n```"
        else:
            return "---"

    result = []
    for element in elements:
        mer = element.get("mermaidGraph", "---")
        if mer and mer != "---":
            norm_mermaid = _normalize_mermaid_graph(mer)
            mer_out = f"\n\n```mermaid\n{norm_mermaid}\n```"
        else:
            mer_out = "---"
        result.append(mer_out)
    return result

def extract_basic_dict(elements: Union[Dict, List[Dict]]) -> Union[Dict, List[Dict]]:
    """
    Extract basic dictionary data from elements.

    Args:
        elements: Dictionary or list of dictionaries containing element data

    Returns:
        Dictionary or list of dictionaries with extracted data
    """
    if isinstance(elements, dict):
        body = {'guid': elements['elementHeader']['guid']}
        for key in elements['properties']:
            body[key] = elements['properties'][key]

        # Add classifications if present
        classifications = elements['elementHeader'].get('classifications', [])
        if classifications:
            classification_names = "["
            for classification in classifications:
                if len(classification_names) > 1:
                    classification_names += ", "
                classification_names += f"{classification['classificationName']}"
            body['classification_names'] = classification_names + ']'

        return body

    result = []
    for element in elements:
        if element is None:
            continue
        body = {'guid': element['elementHeader']['guid']}
        for key in element['properties']:
            body[key] = element['properties'][key]

        # Add classifications if present
        classifications = element['elementHeader'].get('classifications', [])
        if classifications:
            classification_names = "["
            for classification in classifications:
                if len(classification_names) > 1:
                    classification_names += ", "
                classification_names += f"{classification['classificationName']}"
            body['classifications'] = classification_names + ']'

        result.append(body)
    return result

def _extract_default_properties(element: dict, columns_struct: dict) -> dict:
    props = element.get('properties', {}) or {}
    normalized = {
        'properties': props,
        'elementHeader': element.get('elementHeader', {}),
    }
    # Common population pipeline
    col_data = populate_common_columns(element, columns_struct)
    columns_list = col_data.get('formats', {}).get('attributes', [])

    return col_data


def _generate_default_output(elements: dict | list[dict], search_string: str,
                             element_type_name: str | None,
                             output_format: str = 'DICT',
                             report_format: dict | str | None = None,
                             **kwargs) -> str | list[dict]:
    entity_type = 'Referenceable' if element_type_name is None else element_type_name
    # Backward compatibility: accept legacy kwarg
    if report_format is None and isinstance(kwargs, dict) and 'report_spec' in kwargs:
        report_format = kwargs.get('report_spec')
    if report_format:
        if isinstance(report_format, str):
            output_formats = select_report_format(report_format, output_format)
        elif isinstance(report_format, dict):
            output_formats = get_report_spec_match(report_format, output_format)
        else:
            output_formats = None
    else:
        output_formats = select_report_format(entity_type, output_format)
    if output_formats is None:
        output_formats = select_report_format('Default', output_format)
    return generate_output(
        elements=elements,
        search_string=search_string,
        entity_type=entity_type,
        output_format=output_format,
        extract_properties_func=_extract_default_properties,
        get_additional_props_func=None,
        columns_struct=output_formats,
    )


def _normalize_mermaid_graph(mermaid_code: str) -> str:
    """Normalize and transform Mermaid graph for broad compatibility.
    - Ensures standard line endings (\n)
    - Escapes literal newlines in double-quoted labels
    - Converts newer @{ shape: ..., label: ... } syntax to standard nodes
    - Removes frontmatter (---) which is often unsupported in older versions
    """
    if not mermaid_code or not isinstance(mermaid_code, str):
        return mermaid_code

    if not NORMALIZE_MERMAID:
        return mermaid_code

    # 1. Basic normalization
    mermaid_code = mermaid_code.strip().replace("\r\n", "\n").replace("\r", "\n")

    # 2. Handle frontmatter (move title out of Mermaid block if present)
    if mermaid_code.startswith("---"):
        parts = mermaid_code.split("---", 2)
        if len(parts) >= 3:
            mermaid_code = parts[2].strip()

    # 3. Escape literal newlines in labels
    if '"' in mermaid_code:
        mermaid_code = re.sub(
            r'"(?:[^"\\]|\\.)*"',
            lambda m: m.group(0).replace("\n", "\\n"),
            mermaid_code,
        )

    # 4. Syntax Compatibility Transformation
    def transform_shape(match):
        node_id = match.group(1)
        shape_type = match.group(2).lower()
        label = match.group(3)

        if shape_type in ("rounded", "stadium"):
            return f"{node_id}({label})"
        elif shape_type in ("diamond", "decision"):
            return f"{node_id}{{{label}}}"
        elif shape_type in ("circle",):
            return f"{node_id}(({label}))"
        elif shape_type in ("hexagon",):
            return f"{node_id}{{{{{label}}}}}"
        else:
            # Default to square box for unknown/doc/etc.
            return f"{node_id}[{label}]"

    # Regex to match: node_id@{ shape: some_shape, label: "some label" }
    mermaid_code = re.sub(
        r"(\w+)\s*@\{\s*shape:\s*(\w+),\s*label:\s*(\"(?:[^\"\\]|\\.)*\")\s*\}",
        transform_shape,
        mermaid_code,
    )

    return mermaid_code


def _lg_node_title(d: Dict[str, Any]) -> str:
    return (
        d.get("displayName")
        or d.get("name")
        or d.get("qualifiedName")
        or d.get("guid")
        or "(unnamed)"
    )


def _lg_rel_label(d: Dict[str, Any]) -> str:
    try:
        return (
            (d.get("relationshipHeader") or {}).get("type", {}).get("typeName")
            or "related"
        )
    except Exception:
        return "related"


def _lg_extract_peers(norm: Dict[str, Any]) -> list[Dict[str, Any]]:
    peers: list[Dict[str, Any]] = []
    for k, v in norm.items():
        if k == "nested_elements":
            continue
        if isinstance(v, list) and v and all(isinstance(i, dict) for i in v):
            # Treat lists that carry a relationshipHeader as peer links
            if any(isinstance(i.get("relationshipHeader"), dict) for i in v):
                peers.extend(v)
        # Some materializations may have already flattened peer items without explicit key markers
    return peers


def _lg_extract_children(norm: Dict[str, Any]) -> list[Dict[str, Any]]:
    items = norm.get("nested_elements")
    if isinstance(items, list) and all(isinstance(i, dict) for i in items):
        return items
    return []


def generate_linked_graph_report(
    elements: Union[Dict[str, Any], List[Dict[str, Any]]],
    *,
    entity_type: str = "Referenceable",
    max_depth: int = 5,
    include_peers: bool = True,
    include_children: bool = True,
) -> str:
    """
    Produce a recursive, linked Markdown report that walks peers and children.
    - Adds anchors by GUID and intra-doc links for peer/child elements
    - Renders each GUID exactly once; later references are links only
    """
    items: List[Dict[str, Any]] = elements if isinstance(elements, list) else [elements]
    visited: set[str] = set()
    lines: list[str] = []

    def render_node(raw: Dict[str, Any], depth: int = 0) -> None:
        norm = materialize_egeria_summary(raw)
        guid = norm.get("guid") or f"no-guid-{id(norm)}"
        title = _lg_node_title(norm)

        if guid in visited:
            return

        visited.add(guid)
        # Section header and identifiers
        if lines:
            lines.append("\n---\n")
        lines.append(
            f'# {norm.get("type", entity_type)} Name: {title} <a id="{guid}" name="{guid}"></a>\n'
        )
        qn = norm.get("qualifiedName")
        if qn:
            lines.append(f"- Qualified Name: `{qn}`")
        lines.append(f"- GUID: `{guid}`\n")

        # Scalar properties (skip big nested/header keys)
        keys_to_skip = {"guid", "type", "name", "displayName", "qualifiedName", "nested_elements"}
        props = []
        mermaid_blocks = []
        for k, v in norm.items():
            if k in keys_to_skip:
                continue

            # Special handling for Mermaid graphs
            if isinstance(v, str) and ("flowchart" in v.lower() or "mermaid" in k.lower()):
                mermaid_blocks.append((k, v))
                continue

            if isinstance(v, (dict, list)):
                continue
            props.append((k, v))

        if props:
            lines.append("### Properties")
            for k, v in sorted(props, key=lambda kv: kv[0].lower()):
                lines.append(f"- {k}: {v}")
            lines.append("")

        for k, v in mermaid_blocks:
            lines.append(f"### {camel_to_title_case(k)}")
            lines.append("```mermaid")
            # Normalize and transform Mermaid graph for compatibility
            mermaid_code = _normalize_mermaid_graph(v)
            lines.append(mermaid_code)
            lines.append("```\n")

        if depth >= max_depth:
            return

        peers = _lg_extract_peers(norm) if include_peers else []
        children = _lg_extract_children(norm) if include_children else []

        if peers:
            lines.append("### Peers")
            for p in peers:
                pn = materialize_egeria_summary(p)
                plabel = _lg_rel_label(p)
                lines.append(f"- [{_lg_node_title(pn)}](#{pn.get('guid')}) — {plabel}")
            lines.append("")

        if children:
            lines.append("### Children")
            for c in children:
                cn = materialize_egeria_summary(c)
                lines.append(f"- [{_lg_node_title(cn)}](#{cn.get('guid')})")
            lines.append("")

        # Recurse
        for nxt in list(peers) + list(children):
            render_node(nxt, depth + 1)

    for el in items:
        if isinstance(el, dict):
            render_node(el, 0)

    return "\n".join(lines)


def generate_output(elements: Union[Dict, List[Dict]],
               search_string: Optional[str] = None,
               entity_type: str = "Referenceable",
               output_format: str = "DICT",
               extract_properties_func: Optional[Callable] = None,
               get_additional_props_func: Optional[Callable] = None,
               columns_struct: dict = None,
               include_preamble: bool = True,
               **kwargs) -> Union[str, list[dict]]:
    """
    Generate output in the specified format for the given elements.

    Args:
        elements: Dictionary or list of dictionaries containing element data
        search_string: The search string used to find the elements
        entity_type: The type of entity (e.g., "Glossary", "Term", "Category")
        output_format: The desired output format (MD, FORM, REPORT, LIST, DICT, MERMAID, HTML)
        extract_properties_func: Function to extract properties from an element
        get_additional_props_func: Optional function to get additional properties
        columns_struct: Optional report specification structure
        include_preamble: Whether to include the report header/preamble
        **kwargs: Additional arguments, including potential 'filter_string'

    Returns:
        Formatted output as string or list of dictionaries
    """
    if search_string is None:
        search_string = kwargs.get('filter_string')

    columns = columns_struct['formats'].get('attributes',None) if columns_struct else None
    if not columns:
        columns_struct = select_report_format("Default",output_format)
        if columns_struct:
            columns = columns_struct.get('formats', {}).get('attributes', None)

    target_type = columns_struct.get('target_type', entity_type) if columns_struct else entity_type
    if target_type is None:
        target_type = entity_type

    # Ensure elements is a list
    if isinstance(elements, dict):
        elements = [elements]

    # Handle empty search string
    if search_string is None or search_string == '':
        search_string = "All"

    # Set the output format to DICT to return values to table display
    # if output_format == "TABLE":
    #     output_format = "DICT"

    # Generate output based on format
    if output_format.upper() == 'JSON':
        return elements

    if output_format == 'MERMAID':
        return extract_mermaid_only(elements)

    elif output_format == 'HTML':
        # First generate the REPORT format output
        report_output = generate_output(
            elements=elements,
            search_string=search_string,
            entity_type=entity_type,
            output_format="REPORT",
            extract_properties_func=extract_properties_func,
            get_additional_props_func=get_additional_props_func,
            columns_struct=columns_struct
        )

        # Convert the markdown to HTML
        return markdown_to_html(report_output)

    elif output_format in ['DICT', 'TABLE']:
        return generate_entity_dict(
            elements=elements,
            extract_properties_func=extract_properties_func,
            get_additional_props_func=get_additional_props_func,
            exclude_keys=['properties'],
            columns_struct=columns_struct,
            output_format=output_format
        )

    elif output_format == 'LIST':
        if columns is None:
            raise ValueError("Columns must be provided for LIST output format")

        return generate_entity_md_table(
            elements=elements,
            search_string=search_string,
            entity_type=entity_type,
            extract_properties_func=extract_properties_func,
            columns_struct=columns_struct,
            get_additional_props_func=get_additional_props_func,
            output_format=output_format
        )

    elif output_format == 'REPORT-GRAPH':
        return generate_linked_graph_report(
            elements=elements,
            entity_type=target_type,
            max_depth=kwargs.get('max_depth', 5),
            include_peers=kwargs.get('include_peers', True),
            include_children=kwargs.get('include_children', True),
        )

    else:  #  MD, FORM, REPORT
        elements_md = ""
        elements_action = "Details"
        if include_preamble:
            elements_md, elements_action = make_preamble(
                obj_type=target_type,
                search_string=search_string,
                output_format=output_format
            )

        elements_md += generate_entity_md(
            elements=elements,
            elements_action=elements_action,
            output_format=output_format,
            entity_type=target_type,
            extract_properties_func=extract_properties_func,
            get_additional_props_func=get_additional_props_func,
            columns_struct = columns_struct
        )

        return elements_md
