"""
all the table models are added here for p8 schema and can be iterated and used to generate install scripts
we add system fields in the sql model such as timestamps and user ids
"""

import uuid
from pydantic import Field, model_validator
from pydantic.fields import FieldInfo
import typing
from ..AbstractModel import AbstractModel, BaseModel, AbstractEntityModel
from .. import inspection
from percolate.utils import make_uuid
import datetime
from .. import DefaultEmbeddingField, KeyField
from percolate.utils.names import EmbeddingProviders
import percolate as p8
import json
from percolate.utils import get_iso_timestamp
import hashlib
from enum import Enum
#we shouldnt really need a logger her but we have temp functions being exported to apis
from percolate.utils import logger
class Function(AbstractEntityModel):
    """Functions are external tools that agents can use. See field comments for context.
    Functions can be searched and used as LLM tools. 
    The function spec is derived from OpenAPI but adapted to the conventional used in LLMs
    """
         
    id: uuid.UUID | str = Field(description="A unique id in this case generated by the proxy and function name") 
    key: typing.Optional[str] = Field(description='optional key e.g operation id')
    name: str = Field(description="A friendly name that is unique within the proxy scope e.g. a single api or python library")
    verb:  typing.Optional[str] = Field(None,description= "The verb e.g. get, post etc")
    endpoint: typing.Optional[str] = Field(None, description="A callable endpoint in the case of REST")
    description: str = DefaultEmbeddingField('', description="A detailed description of the function - may be more comprehensive than the one within the function spec - this is semantically searchable")
    function_spec: dict = Field(description="A function description that is OpenAI and based on the OpenAPI spec")
    proxy_uri: str = Field(description='a reference to an api or library namespace that qualifies the named function')
    
    def __call__(self, *args, **kwargs):
        """
        Convenience to call any function via its proxy
        """
        return p8.get_proxy(self.proxy_uri).invoke(self, **kwargs)
        
    @classmethod
    def from_entity(cls, model:BaseException):
        """The function produces a callable agent stub that can be searched.
           In the database we use a native generic function handler and in python we load the agent from the name to call the agent.
           Agents are discoverable in this way and we can transfer context to other agents.
        """
        M:AbstractModel = AbstractModel.Abstracted(model)
    
        description = f"""
        ## Agent: {M.get_model_full_name()}
        ## Description
        ```
        {M.get_model_description()}
        ```
        ## Functions
        ```json
        {M.get_model_functions()}
        ```
        """
        
        """this must be the same interface as the Agent interface i.e. we relay to the agent.run(question='')"""
        def run(question:str):
            """
            send any query or request to the agent to execute the agent
            
            Args:
                question: a request to the agent
            """
            pass
        
        key = f"{M.get_model_full_name()}.run".replace('.','_')
        
        """the alias is important because we need to qualify functions used by agents and the proxy above is just a signature hint"""
        spec = cls.from_callable(run,alias=key).function_spec
        """names should not have '.'"""
        
        """for the name we qualify with run agent so its clear what we are doing"""
        return Function(name=key, 
                        key=key,
                        description=description,
                        function_spec=spec,
                        verb='get',
                        proxy_uri=f'p8agent/{M.get_model_full_name()}', #handler can use this - a p8_agent/agent.name - see interface proxy manager
                        endpoint='run')
        
    @classmethod
    def from_callable(cls, fn:typing.Callable, remove_untyped:bool=True, proxy_uri:str=None,alias:str=None):
        """construct function from callable - supply an alias if the function is not full qualified or if we need extra context"""
        def process_properties(properties: dict):
             
            untyped = []
            """google at least wont like some of these but they are redundant anyway"""
            for key, details in properties.items():
                for remove_field in ['title', 'default']:
                    if remove_field in details:
                        details.pop(remove_field)
       
                if 'anyOf' in details:
                    new_list = [t for t in details['anyOf'] if t['type']!= 'null']
                   
                    #temp ignore anything after the first type
                    if len(new_list) >= 1:
                        details['type'] = new_list[0]['type']
                        details.pop('anyOf')
                    else:
                        details.pop('anyOf')
                        details['oneOf'] = new_list
               
                if 'properties' in details:
                    process_properties(details['properties'])
                
                """for language models there is no point sending untyped information
                TODO: handle this better e.g. typed kwargs - specifying additional properties true might be enough but its unclear how to handle them anyway
                """
                if not details and remove_untyped:
                    untyped.append(key)
                

            
        def _map(f):
            """make sure the structure from pydantic is the same as used elsewhere for functions"""
            p = dict(f)
            if 'properties' in p:
                process_properties(p['properties'])
    
            name = p.pop('title')
            desc = p.pop('description') if 'description' in p else 'NO DESC'
            return {
                'name': alias or name,
                'parameters' :p,
                'description': desc
            }
        
        """we can pass this in e.g. for native. Lib is used for lib runtime or APIs for REST"""
        if not proxy_uri:
            proxy_uri = fn.__self__ if hasattr(fn, '__self__') else 'lib'
            if not isinstance(proxy_uri,str):
                proxy_uri = inspection.get_object_id(proxy_uri)
            
        s = _map(AbstractModel.create_model_from_function(fn).model_json_schema())
        key = s['name'] if not proxy_uri else f"{proxy_uri}.{s['name']}"
        id_md5_uuid =  uuid.uuid3(uuid.NAMESPACE_DNS, key)   
        return cls(id=str(id_md5_uuid), 
                   name = s['name'],
                   key  = key, 
                   endpoint=s['name'],
                   verb='get',
                   proxy_uri=proxy_uri,
                   spec=s, 
                   function_spec=s,
                   description=s['description'],
                   #this is ignored by Function but the runtime function can carry it
                   fn=fn)
        
    @model_validator(mode='before')
    @classmethod
    def _f(cls, values):
        if not values.get('id'):
            values['id'] = make_uuid({ 'key': values['name'], 'proxy_uri':values['proxy_uri']})
        return values
        
   
class ApiProxy(AbstractEntityModel):
    """A list of proxies or APIs that have attached functions or endpoints - links to proxy_uri on the Function""" 
    id: typing.Optional[uuid.UUID | str] = Field(None, description="Will default to a hash of the uri")
    name: typing.Optional[str] = Field(None, description="A unique api friendly name")
    proxy_uri: str = Field(description='a reference to an api or library namespace that qualifies the named function')
    token: typing.Optional[str] = Field(None, description="the token to save")
    
            
    @model_validator(mode='before')
    @classmethod
    def _f(cls, values):
        if not values.get('name'):
            values['name'] = values['proxy_uri']
        if not values.get('id'):
            values['id'] = make_uuid(values['proxy_uri'])
        return values
        
    
    
class LanguageModelApi(AbstractEntityModel):
    """The Language model REST Apis are stored with tokens and scheme information.
    """
    
    id: uuid.UUID | str
    name: str = Field(description="A unique name for the model api e.g cerebras-llama3.1-8b")
    model: typing.Optional[str] = Field(None, description="The model name defaults to the name as they are often the same. the name can be unique based on a provider qualfier")
    scheme: typing.Optional[str] = Field('openai', description="In practice most LLM APIs use an openai scheme - currently `anthropic` and `google` can differ")
    completions_uri: str = Field(description="The api used for completions in chat and function calling. There may be other uris for other contexts")
    token_env_key: typing.Optional[str] = Field(None, description="Conventions are used to resolve keys from env or other services. Provide an alternative key")
    token: typing.Optional[str] = Field(None, description="It is not recommended to add tokens directly to the data but for convenience you might want to")
    
        
    @model_validator(mode='before')
    @classmethod
    def _f(cls, values):
        if not values.get('model'):
            values['model'] = values['name']
        return values
        
    
class Agent(AbstractEntityModel):
    """The agent model is a meta data object to persist agent metadata for search etc"""

    id: uuid.UUID| str  
    name: str
    category: typing.Optional[str] = Field(None,description="Simple property to filter agents by categories")
    description: str = DefaultEmbeddingField(description="The system prompt as markdown")
    spec: dict = Field(description="The model json schema")
    functions: typing.Optional[dict] = Field(description="The function that agent can call",default_factory=dict)
    
    @model_validator(mode='before')
    @classmethod
    def _f(cls, values):
        """we take these from the class and save them"""
        if not values.get('functions') and hasattr(cls, 'get_model_functions'):
            values['functions'] = cls.get_model_functions()
        return values
    
    
    def from_abstract_model(cls: BaseModel):
        """Given any pydantic model that behaves like an AbstractModel, get the meta object i.e. Agent"""
        cls:AbstractModel = AbstractModel.Abstracted(cls)
        name = cls.get_model_full_name()
        return Agent(id=make_uuid(name), 
                     name=name, 
                     functions=cls.get_model_functions(),
                     spec = cls.model_json_schema(),
                     description=cls.get_model_description())

class ModelField(AbstractEntityModel):
    """Fields are each field in any saved model/agent. 
    Fields are useful for describing system info such as for embeddings or for promoting.
    """
            
    id: typing.Optional[uuid.UUID|str] = Field(description="a unique key for the field e.g. field and entity key hashed")
    name: str = Field(description="The field name")
    entity_name: str
    field_type: str
    embedding_provider: typing.Optional[str] = Field(None, description="The embedding could be a multiple in future")
    description: typing.Optional[str] = None
    is_key: typing.Optional[bool] = Field(default=False, description="Indicate that the field is the primary key - our convention is the id field should be the primary key and be uuid and we use this to join embeddings")
    
    @model_validator(mode='before')
    @classmethod
    def _f(cls, values):
        """this is a convenience for 'dont care' cases where you want an embedding but dont know the model name"""
        if values.get('embedding_provider') == 'default':
            values['embedding_provider'] = EmbeddingProviders.embedding_text_embedding_ada_002
        if not values.get('id'):
            values['id'] = make_uuid({'name': values['name'], 'entity_name': values['entity_name']})
        return values
    
    @classmethod
    def from_field_info(cls, key:str, field_type: FieldInfo, parent_model:BaseModel):
        """five a FieldInfo object map the field"""
        
        parent_model:AbstractModel = AbstractModel.Abstracted(parent_model)
        json_args = field_type.json_schema_extra or {}
        arg = inspection.get_innermost_args(field_type.annotation)
        arg = str(arg.__name__) if hasattr(arg, '__name__') else str(arg)
        return ModelField(name=key, 
                           entity_name= parent_model.get_model_full_name(), 
                           field_type= arg,
                           description=field_type.description,
                           **json_args)
        
    @staticmethod
    def get_fields_from_model(cls:BaseModel):
        """for any pydantic model generate the field collection"""
        cls = AbstractModel.Abstracted(cls)
        return [ModelField.from_field_info(key, value,parent_model=cls) for key, value in cls.model_fields.items()]
    
 

class TokenUsage(AbstractModel):
    """Tracks token usage for language model interactions"""
    model_config = {'protected_namespaces': ()}
    id: uuid.UUID| str  
    model_name: str
    tokens: typing.Optional[int] = Field(0,description="the number of tokens consumed in total")
    tokens_in: typing.Optional[int] = Field(0, description="the number of tokens consumed for input")
    tokens_out: typing.Optional[int] = Field(0, description="the number of tokens consumed for output")
    tokens_other: typing.Optional[int] = Field(0, description="the number of tokens consumed for functions and other metadata")
    session_id: typing.Optional[uuid.UUID| str  ] = Field(None, description="Session id for a conversation")
    
    @model_validator(mode='before')
    @classmethod
    def _f(cls, values):
        if not values.get('tokens'):
            values['tokens'] = (values.get('tokens_in') or 0) + (values.get('tokens_out') or 0)
        return values
    
class IndexAudit(TokenUsage):
    model_config = {
        'index_notify': False        
    }
    """Track requests to build smart indexes such as graph links or text and image embeddings"""
    metrics: typing.Optional[dict] = Field(description="metrics for records indexed",default_factory=dict)
    status: str = Field(description="Status code such as OK|ERROR")
    message: typing.Optional[str] = Field(description="Any message such as an error")
    entity_full_name: str
    
    
class _OpenAIMessage(BaseModel):
    role: str
    content: typing.Optional[str] = Field('', description='text content')
    tool_calls: typing.Optional[typing.List[dict]|dict]
    tool_call_id: typing.Optional[str] = None
    
    @classmethod
    def from_message(cls, values):
        """
        
        """
        return _OpenAIMessage(**values, tool_call_id=(values.get('tool_eval_data') or {}).get('id'))
    
    @model_validator(mode='before')
    @classmethod
    def _val(cls,values):
        if tool_calls:= values.get('tool_calls'):
            if isinstance(tool_calls,dict):
                values['tool_calls'] = [tool_calls]
        return values

class AIResponse(TokenUsage):
    """Each atom in an exchange between users, agents, assistants and so on. 
    We generate questions with sessions and then that triggers an exchange. 
    Normally the Dialogue is round trip transaction.
    """
    model_config = {'protected_namespaces': ()}
    id: uuid.UUID| str  
    role: str = Field(description="The role of the user/agent in the conversation")
    content: str = DefaultEmbeddingField(description="The content for this part of the conversation") #TODO we may not want to automatically generate embeddings for this table
    status: typing.Optional[str] = Field(description="The status of the session such as REQUEST|RESPONSE|ERROR|TOOL_CALL|STREAM_RESPONSE")
    tool_calls: typing.Optional[typing.List[dict]|dict] = Field(default=None, description="Tool calls are requests from language models to call tools")
    tool_eval_data: typing.Optional[dict] = Field(default=None, description="The payload may store the eval from the tool especially if it is small data")
    verbatim: typing.Optional[dict|typing.List[dict]] = Field(default=None, description="the verbatim message from the language model - we dont serialize this", exclude=True)     
    function_stack: typing.Optional[typing.List[str]] = Field(None, description='At each stage certain functions are available to the model - useful to see what it has and what it chooses and to reload stack later')
 
    def to_open_ai_message(self):
        """the message structure for the scheme
        the one thing we need to do is make sure that tool calls reference the id(s)
        """
        return _OpenAIMessage.from_message(self.model_dump())
    
    @classmethod
    def from_open_ai_response(cls, response, sid:str)  :
        """"""
        choice = response['choices'][0]
        usage = response['usage']
        message = choice['message']
        tool_calls= message.get('tool_calls')
        return cls(
            id=str(uuid.uuid1()),
            role=message.get('role'),
            tool_calls=tool_calls,
            content=message.get('content') or '',
            tokens_in=usage['prompt_tokens'],
            tokens_out=usage['completion_tokens'],
            model_name=response['model'],
            status='TOOL_CALL' if not tool_calls else 'RESPONSE',
            session_id=sid
        )
class Session(AbstractModel):
    """Tracks groups if session dialogue - sessions are requests that produce actions or activity. 
    Typical example is interacting with an AI but also it could be simply a request to bookmark or ingestion some content.
    From the user's perspective a session is a single interaction but internally the system performs multiple hops in general - hence "session"
    Sessions can be interactions with the system or agent but they can be inferred e.g. from user activity
    Sessions can also be abstract for example a daily diary can be an abstract session with the intent of capturing daily activity
    Sessions can be hierarchical and moments can be generated from sessions.
    
    Sessions model user intent.
    """
    id: uuid.UUID| str  
    """i should maybe deprecate the name here as this is a dense audit but then maybe we should just archive?"""
    name: typing.Optional[str] = Field(None, description="The name is a pseudo name to make sessions node-compatible")
    query: typing.Optional[str] = DefaultEmbeddingField(None,description='the question or context that triggered the session - query can be an intent and it can be inferred')
    user_rating: typing.Optional[float] = Field(None, description="We can in future rate sessions to learn what works")
    agent: str = Field('percolate', description="Percolate always expects an agent but we support passing a system prompt which we treat as anonymous agent")
    parent_session_id:typing.Optional[uuid.UUID| str] = Field(None, description="A session is a thread from a question+prompt to completion. We span child sessions")
    
    thread_id: typing.Optional[str] = Field(None,description='An id for a thread which can contain a number of sessions - thread matches are case insensitive ie.. MyID==myid - typically we prefer ids to be uuids but depends on the system')
    channel_id: typing.Optional[str] = Field(None, description="The platform/channel ID through which the user is interacting (e.g., specific Slack channel ID)")
    channel_type: typing.Optional[str] = Field('percolate', description="The platform type (e.g., 'slack', 'percolate', 'email') - the device info + the channel type tells if its mobile etc")
    session_type: typing.Optional[str] = Field('conversation', description="The type of session (e.g., 'conversation', 'resource_management', 'task_creation', 'daily_digest')")

    metadata: typing.Optional[dict] = Field(default_factory=dict, description="Arbitrary metadata - typically this is device info and other context")
    session_completed_at: typing.Optional[datetime.datetime] = Field(default=None,description="An audit timestamp to write back the completion of the session - its optional as it should manage the last timestamp on the AI Responses for the session")
    graph_paths: typing.Optional[typing.List[str]] = Field(None, description="Track all paths extracted by an agent as used to build the KG over user sessions")
    userid:  typing.Optional[uuid.UUID| str ] = Field(None,description="The user id to use") #implicit
        
    
    @model_validator(mode='before')
    @classmethod
    def _f(cls, values): 
        if isinstance(values,dict) and not values.get('id'):
            """the name is mapped for a hash of the query and can create collisions for now since we only want to index intent"""
            # TODO - consider if this is ok ^
            values['name'] = make_uuid({'query': values.get('query', '')})
        return values
    
    @classmethod
    def get_daily_diary_thread_id(cls, userid:str,  session_type:str=None):
        """this is a simple hash for filtering and respect the contract for threads"""
        daily_diary_id = make_uuid({"user_id": userid, "session_type": session_type})
        return daily_diary_id
    
    @classmethod
    def daily_diary_entry(cls, userid:str, query:str, metadata:dict=None, session_type:str='daily_digest'):
        """
        generates a task for the daily diary
        the query should be the intent e.g. a question from a user or an action such as upload file
        """
        
        return Session(id = str(uuid.uuid1()), 
                       query=query,
                       name =f"Diary entry {get_iso_timestamp()}",
                       session_type=session_type, 
                       userid=userid,
                       thread_id=cls.get_daily_diary_thread_id(userid=userid,session_type=session_type), 
                       metadata=metadata)
    
    @classmethod
    def task_thread_entry(cls, thread_id :str,  userid:str, query:str, metadata:dict=None,session_type:str='daily_digest'):
        """
        generates a task for the daily diary
        the query should be the intent e.g. a question from a user or an action such as upload file
        """
        
        return Session(id = str(uuid.uuid1()), 
                       query=query,
                       name =f"Diary entry {get_iso_timestamp()}",
                       session_type=session_type, 
                       userid=userid,
                       thread_id=thread_id, 
                       metadata=metadata)
    
    
    
    
    @classmethod
    def from_question_and_context(cls, id:str, question:str, context:typing.Any, agent:str=None, **kwargs):
        """
        from the session we can track the question and the agent that serviced. we may also want to store other details 
        """
        from percolate.services.llm import CallingContext
        context: CallingContext = context
        
        #TODO: we should capture some of context e.g. threads and users 
        return cls(query=question,
            id=id or str(uuid.uuid1()), 
            agent=agent,
            #TODO user id and channel contexts etc.
            **kwargs)
         
        
class SessionEvaluation(AbstractModel):
    """Tracks groups if session dialogue"""
    id: uuid.UUID| str  
    rating: float = Field(None,description="A rating from 0 to 1 - binary thumb-up/thumbs-down are 0 or 1")
    comments: typing.Optional[str] = Field(None, description="Additional feedback comments from the user")
    session_id: uuid.UUID| str  
    
class ModelMatrix(AbstractModel):
    """keep useful json blobs for model info"""
    id: uuid.UUID| str  
    entity_name: str = Field(description="The name of the entity e.g. a model in the types or a user defined model")
    enums: typing.Optional[dict] = Field(None, description="The enums used in the model - usually a job will build these and they can be used in query prompts")
    example_sql_queries: typing.Optional[dict] = Field(None, description="A mapping of interesting question to SQL queries - usually a job will build these and they can be used in query prompts")

class Category(AbstractEntityModel):
    """A category is just a general topic node that can be linked to other nodes and can have an evolving description or summary"""
    id: uuid.UUID| str  
    name: str = Field(description="The name of the category node")
    description: str = DefaultEmbeddingField(description="The content description for the category")
    
class Project(AbstractEntityModel):
    """A project is a broadly defined goal with related resources (uses the graph)"""
    id: uuid.UUID| str  
    name: str = Field(description="The name of the entity e.g. a model in the types or a user defined model")
    description: str = DefaultEmbeddingField(description="The content for this part of the conversation")
    target_date: typing.Optional[datetime.datetime] = Field(None, description="Optional target date")
    collaborator_ids: typing.List[uuid.UUID] = Field(default_factory=list, description="Users collaborating on this project")
    status: typing.Optional[str] = Field(default="active", description="Project status")
    priority: typing.Optional[int] = Field(default=1, description="Priority level (1-5), 1 being the highest priority")
    
class Task(Project):
    """Tasks are sub projects. A project can describe a larger objective and be broken down into tasks.
If if you need to do research or you are asked to search or create a research iteration/plan for world knowledge searches you MUST ask the _ResearchIteration agent_ to help perform the research on your behalf (you can request this agent with the help i.e. ask for an agent by name). 
You should be clear when relaying the users request. If the user asks to construct a plan, you should ask the agent to construct a plan. If the user asks to search, you should ask the research agent to execute  a web search.
If the user asks to look for existing plans, you should ask the research agent to search research plans.
    """
    id: typing.Optional[uuid.UUID| str] = Field(None,description= 'id generated for the name and project - these must be unique or they are overwritten')
    project_name: typing.Optional[str] = Field(None, description="The related project name if relevant")
    estimated_effort: typing.Optional[float] = Field(default=None, description="Estimated effort in hours")
    progress: typing.Optional[float] = Field(default=0.0, description="Completion progress (0-1)")
    
    @model_validator(mode='before')
    @classmethod
    def _f(cls, values): 
        if isinstance(values,dict) and not values.get('id'):
            values['id'] = make_uuid({'name': values['name'], 'project_name': values.get('project_name') or ''})
        return values
    
    @classmethod
    def get_model_functions(cls):
        """fetch task external functions"""
        return {'post_tasks_': 'Used to save tasks by posting the task object. Its good practice to first search for a task of a similar name before saving in case of duplicates' }
    
class User(AbstractEntityModel):
    """A model of a logged in user"""
    id: uuid.UUID| str  
    name: typing.Optional[str] = Field(None,description="A display name for a user")
    email: typing.Optional[str] = KeyField(None,description="email of the user if we know it - the key must be something unique like this")
    slack_id: typing.Optional[str] = Field(None,description="slack user U12345")
    linkedin: typing.Optional[str] = Field(None,description="linkedin profile for user discovery")
    twitter: typing.Optional[str] = Field(None,description="twitter profile for user discovery")

    description: typing.Optional[str] = DefaultEmbeddingField(default='', description="A learned description of the user")
    #current_chat_thread: typing.Optional[typing.List[str]] = Field(default_factory=list, description="A thread is a single conversation on any channel - the session contains a list of messages with thread id binding them")
    recent_threads: typing.Optional[typing.List[dict]|dict] = Field(default_factory=list, description="A thread is a single conversation on any channel - the session contains a list of messages with thread id binding them")
    last_ai_response: typing.Optional[str] = Field(None, description="We store this context for managing conversations and state")
    interesting_entity_keys: typing.Optional[dict] = Field(default_factory=dict, description="For the agents convenience a short term mapping of interesting keys with optional descriptions based on user activity")
    token: typing.Optional[str] = Field(None, description="A token for user authentication to Percolate")
    roles: typing.Optional[typing.List[str]] = Field(default_factory=list, description="A list of roles the user is a member of")
    graph_paths: typing.Optional[typing.List[str]] = Field(None, description="Track all paths extracted by an agent as used to build the KG")
    metadata: typing.Optional[dict] = Field(default_factory=dict, description="Arbitrary user metadata")
    email_subscription_active: typing.Optional[bool] = Field(False, description="Users can opt in and out of emails")
    
    @staticmethod
    def id_from_email(email):
        return make_uuid(email)
    
    def as_memory(self,**kwargs):
        """the user memory structure"""

        return {
            'Info': "You can use the users context - observe the current chat thread which may be empty when deciding if the user is referring to something they discussed recently or a new context."
                    "When you do use this context do not explain that to the user as it would be jarring for them. Freely use this context if its relevant or necessary to understand the user context."
                    "The last AI Response from the previous interaction is added for extra context and can be used if the user asks a follow up question in reference to previous response only. But dont ask them for confirmation."
                    "If entity keys are provided you can use the get-entities lookup function to load and inspect them.",
                    "some tools may accept a user id and you can use the user id here for those tool calls e.g. to do user specific searches"
                    
            'recent_threads': self.recent_threads,
            'last_ai_response': self.last_ai_response,
            'interesting_entity_keys': self.interesting_entity_keys,
            'users_name': self.name,
            'about user' : self.description,
            'user_id': self.id
        }
        
class UserFact(AbstractModel):
    """
    important information or trivial about a user typically submitted by a user over time.
    we can search specifically for user submitted data using vector search or entity lookup for labeled data.
    """
    id: uuid.UUID| str  
    name: typing.Optional[str] = Field(None,description="A unique entity name e.g. user_id-label")
    label: typing.Optional[str] = Field(None,description="The friendly label for the piece of information")
    description: typing.Optional[str] = DefaultEmbeddingField(default='', description="A description from the user e.g. my nick name is, my favourite color is, i have traveled to, i like foods like...")
    invalidated: typing.Optional[bool] = Field(False, description="We can store old information but invalidate it - if we retrieve such information we may discard it as relevant")
    graph_paths: typing.Optional[typing.List[str]] = Field(None, description="Track all paths extracted by an agent as used to build the KG")
    userid: str
    
    @classmethod
    def save_user_fact(cls, label:str, description:str, user_id: str, graph_paths:typing.List[str]=None):
        """save the user fact with a unique label for the information
        
        Args:
            description: details about the user fact
            unique_label: a user level unique label about the fact - should not collide with other user facts (best effort)
            user_id: the user id supplied in context- cif not known do not try to use this function
            graph_paths: graph paths are tags of the form A/B where A is more specific than B e.g. LLMs/AI
        """
        name = f"{user_id}.{label}"
        id = make_uuid(name)
        u = UserFact(id=id,name=name, label=label,description=description, graph_paths=graph_paths,userid=user_id)
        return p8.repository(u).update_records(u)
    
    @classmethod
    def search_user_facts(cls, query:str, user_id:str):
        """
        Search semantically filtered by a given user id
        
        Args:
            query: a detailed question to search for
            user_id: str the provided user id if known - if not known, do not try to use this function
        """
        
        """for now just search all but filter by user id assumed"""
        return p8.repository(UserFact).search(query,user_id=user_id)
    
    @classmethod
    def get_user_entity(cls, label:str,user_id:str):
        """looks up a user entity using the label qualified with the user id"""
        name = f"{user_id}.{label}"
        return p8.repository(UserFact).get_entities(name)
        
        
class Resources(AbstractModel):
    """Generic parsed content from files, sites etc. added to the system.
    If someone asks about resources, content, files, general information etc. it may be worth doing a search on the Resources.
    If a user expresses interests or preferences or trivia about themselves - you can save it in the background but response naturally in conversation about it.
    If the user asks information about themselves you can also try to search for user facts if you dont have the answer.
    """
    id: typing.Optional[uuid.UUID| str] = Field("The id is generated as a hash of the required uri and ordinal")  
    name: typing.Optional[str] = Field(None, description="A short content name - non unique - for example a friendly label for a chunked pdf document or web page title")
    category: typing.Optional[str] = Field(None, description="A content category")
    content: str = DefaultEmbeddingField(description="The chunk of content from the source")
    summary: typing.Optional[str] = Field(None,description="An optional summary")
    ordinal: int = Field(0, description="For chunked content we can keep an ordinal")
    uri: str = Field("An external source or content ref e.g. a PDF file on blob storage or public URI")
    metadata: typing.Optional[dict] = {} #for emails it could be sender and receiver info and date
    graph_paths: typing.Optional[typing.List[str]] = Field(None, description="Track all paths extracted by an agent as used to build the KG")
    resource_timestamp: typing.Optional[datetime.datetime] = Field(None,description= 'the database will add a default date but sometimes we want to control when the resource is relevant for - for example a daily log')
    userid:  typing.Optional[uuid.UUID| str] = Field(None, description="The user id is a system field but if we need to control the user context this is how we do it")  
    
    @model_validator(mode='before')
    @classmethod
    def _f(cls, values):
        if not values.get('id'):
            """you need to be careful with the id i.e. ordinals and uris should make the resource unique
            filenames for example should not be used but replaced with e.g. an s3 global uri
            """
            values['id'] = make_uuid({'uri': values['uri'], 'ordinal': values.get('ordinal') or 0})
        return values
    
    @classmethod
    def search_facts_by_users(cls, query:str, user_id:str):
        """
        Search semantically filtered by a given user id
        
        Args:
            query: a detailed question to search for
            user_id: str the provided user id if known - if not known, do not try to use this function
        """
        
        return UserFact.search_user_facts(query, user_id)
    
    @classmethod
    def save_user_fact(cls, description:str, unique_label:str, user_id:str, graph_paths:typing.List[str]):
        """save the user fact with a unique label for the information
        
        Args:
            description: details about the user fact
            unique_label: a user level unique label about the fact - should not collide with other user facts (best effort)
            user_id: the user id supplied in context- if not known do not try to use this function
            graph_paths: graph paths are tags of the form A/B where A is more specific than B e.g. LLMs/AI
        """
        
        return UserFact.save_user_fact(unique_label, description=description,user_id=user_id, graph_paths=graph_paths)

    @classmethod
    def chunked_resource_from_text(cls,
        text: str,
        uri:str,
        chunk_size: int = 1000,
        category: typing.Optional[str] = None,
        name: typing.Optional[str] = None,
        userid: typing.Optional[str] = None
    ) -> typing.List["Resources"]:
        """load text into simple chunked resource"""
        name = name or uri
        chunks = [text[i:i + chunk_size] for i in range(0, len(text), chunk_size)]

        resources = []
        for i, chunk in enumerate(chunks):
            """TODO: NB!!!!!! HERE WE ARE USING URIS WITHOUT PARAMETERS AS GLOBALLY UNIQUE"""
            id_input = f"{uri.split('?')[0]}-{i}"
            id_hash = make_uuid(id_input)

            resource = cls(
                id=id_hash,
                name=f'{name} ({i})' if i > 0 else name,
                category=category,
                content=chunk,
                ordinal=i,
                uri=uri,
                userid=userid
            )
            resources.append(resource)

        return resources
        
        
    @classmethod
    def chunked_resource(
        cls,
        uri: str,
        chunk_size: int = 1000,
        category: typing.Optional[str] = None,
        name: typing.Optional[str] = None,
        userid: typing.Optional[str] = None,
        try_markdown:bool = False
    ) -> typing.List["Resources"]:
        """read file contents from web or file and chunks them by some chunk size - sensible defaults are used
        
        the uri is used as the id but we excluded parameters from the uri
        """
        
        from percolate.utils.parsing import get_content_provider_for_uri
        from pathlib import Path
        
        def sanitize_text(text: str) -> str:
            """this is a dump serializer for now as we work out the data model"""
            import html2text    
            t =  text.replace("\x00", "")  # or use a placeholder like "�"
            if try_markdown:
                try:
                     t = html2text.html2text(t)
                except Exception as ex:
                    #print("HTMLPARSE", ex)
                    pass
            return t
           
        """TODO add an S3 provider that understands signing""" 
        provider = get_content_provider_for_uri(uri)
        text = sanitize_text(provider.extract_text(uri))
        
        return cls.chunked_resource_from_text(text, 
                                              uri, 
                                              chunk_size=chunk_size,
                                              name=name,
                                              userid=userid, 
                                              category=category)

class TaskResources(AbstractModel):
    """(Deprecate)A link between tasks and resources since resources can be shared between tasks"""
    id: typing.Optional[uuid.UUID| str] = Field(None, description="unique id for rel" )  
    resource_id: uuid.UUID| str = Field("The resource id" )  
    session_id:  uuid.UUID| str = Field("The session id is typically a task or research iteration but can be any session id to group resources" )  
    user_metadata: typing.Optional[dict] = Field(default_factory=dict, description="User-specific metadata for this resource")
    relevance_score: typing.Optional[float] = Field(default=None, description="How relevant this resource is to the session")
    @model_validator(mode='before')
    @classmethod
    def _f(cls, values):
        if not values.get('id'):
            """
            """
            values['resource_id'] = str(values['resource_id'])
            values['session_id'] = str(values['session_id'])
            
            values['id'] = make_uuid({'resource_id': values['resource_id'], 'session_id': values.get('session_id') or 0})
            
        return values
    
class SessionResources(AbstractModel):
    """A link between sessions and resources since resources can be shared between sessions"""
    id: typing.Optional[uuid.UUID| str] = Field(None, description="unique id for rel" )  
    resource_id: uuid.UUID| str = Field("The resource id" )  
    session_id:  uuid.UUID| str = Field("The session id is any user intent from a chat/request/trigger" )  
    count: typing.Optional[int] = Field(1, description="In a model where we only store head we can track an estimate for chunk count")
    
    @model_validator(mode='before')
    @classmethod
    def _f(cls, values):
        if not values.get('id'):
            """
            """
            values['resource_id'] = str(values['resource_id'])
            values['session_id'] = str(values['session_id'])
            values['id'] = make_uuid({'resource_id': values['resource_id'], 'session_id': values.get('session_id') or 0})
        return values
class _QuestionSet(BaseModel):
    query:str = Field(description="The question/query to search")
    concept_keys: typing.Optional[typing.List[str]] = Field(default_factory=list, description="A list of codes related to the concept diagram matching questions to the research structure")
    
class ResearchIteration(AbstractModel):
    """
    Your job is to use functions to execute research tasks. You do not search for answers directly, you post a research tasks to generate data.
    A research iteration is a plan to deal with a task.     
    If you are asked for a plan you should first use your json structure to create a plan and give it to the user.

    1. if you are asked to information on general topics, you should execute the post_tasks_research_execute
    2. if you are asked about other research iterations only then can you use the _search_ method which is designed to search research plans as opposed to actually search for general information.
    
    You can generate conceptual diagrams using mermaid diagrams to provide an overview of the research plan.
    When you generate a conceptual plan you should link question sets to plans for example each question should have labels that link to part of the conceptual diagram using the mermaid diagram format to describe your plan.
    """
    id: typing.Optional[uuid.UUID| str] = Field(None, description="unique id for rel" )  
    iteration: int
    content: typing.Optional[str] = DefaultEmbeddingField(None,description="An optional summary of the results discovered")
    conceptual_diagram: typing.Optional[str] = Field(None, description="The mermaid diagram for the plan - typically generated in advanced of doing a search")
    question_set: typing.List[_QuestionSet] = Field(description="a set of questions and their ids from the conceptual diagram")
    task_id: typing.Optional[uuid.UUID| str] = Field(default=None,description="Research are linked to tasks which are at minimum a question" )  
    
    @classmethod
    def get_model_functions(cls):
        """override model functions to provide my available behaviours/tools"""
        return {
            'post_tasks_research_execute': "post the ResearchIteration object to the endpoint to execute a research plan."
        }
        
    """we add some functions for illustration and testing but the database should be used for core functions"""
    # @classmethod
    # def perform_web_search(cls,query:str,limit:int=3)->typing.List[dict]:
    #     """performs a tavily web search - supply a query and receive a set of summaries and urls
        
    #     Args:
    #         query: the long form web query to search using the search api
    #         limit: how many search results to fetch
    #     """
        
    #     from percolate.utils.env import TAVILY_API_KEY
    #     import requests
        
    #     if not TAVILY_API_KEY:
    #         raise Exception(f"The TAVILY_API_KEY key is not set so we have no way to search")
        
    #     headers = { "Authorization": f"Bearer {TAVILY_API_KEY}"  }
        
    #     r  = requests.post("https://api.tavily.com/search", headers=headers, json={'query':query,'limit':limit})
    #     if not r.status_code in [200,201]:
    #         raise Exception(f"Error calling function - {r.content}")
    #     return r.json()
    

    
    @classmethod
    def fetch_web_content(cls,url:str, summarization_context:str=None):
        """provide a uri and fetch the web content - some pages not be accessible and you should typically report the error and try another resource
        
        Args:
            url: the web page to fetch - content will be converted to markdown
            summarization_context: it is recommended to pass a summarization context to reduce propagated tokens - the page will be summarized once in context
        """
        import html2text
        import requests
        headers = {
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36"
        }
        
        data = requests.get(url, headers=headers).content.decode()
        data = html2text.html2text(data)
        if summarization_context:
            data = p8.summarize(data,summarization_context)
        return data
        
class BlockDocument(AbstractModel):
    """Generic parsed content from files, sites etc. added to the system"""
    id: typing.Optional[uuid.UUID| str] = Field("The id is generated as a hash of the required uri and ordinal")  
    name: typing.Optional[str] = Field(None, description="A short content name - non unique - for example a friendly label for a chunked pdf document or web page title")
    category: typing.Optional[str] = Field(None, description="A content category")
    summary: str = DefaultEmbeddingField(description="The chunk of content from the source")
    #conceptual blocks show document structure; inter block links, citations, entities referenced
    content_blocks:typing.List[dict] = Field(description="The content blocks in json format")
    conceptual_diagram: typing.Optional[str] = Field(None, description="The mermaid diagram for the plan or renewed for the document")
    metadata: typing.Optional[dict] = {} #metadata for a document
    
class PercolateAgent(Resources):
    """The percolate agent is the guy that tells you about Percolate which is a multi-modal database for managing AI in the data tier.
    You can learn about the philosophy of Percolate or ask questions about the docs and codebase.
    You can lookup entities of different types or plan queries and searches.
    You can call any registered apis and functions and learn more about how they can be used.
    Call the search function to get data about Percolate
    Only use 'search' if you are asked questions specifically about Percolate otherwise call for help!
    If you do a search and you do not find any data related to the user question you should ALWAYS ask for help to dig deeper.
    """
    @model_validator(mode='before')
    @classmethod
    def _f(cls, values):
        if not values.get('id'):
            values['id'] = make_uuid({'uri': values['uri'], 'ordinal': values['ordinal']})
        return values


class Settings(AbstractModel):
    """settings are key value pairs for percolate admin"""
    id: typing.Optional[uuid.UUID| str] = Field("The id is generated as a hash of the required key and ordinal. The id must be in a UUID format or can be left blank")  
    key: str = Field(default="The key for the value to store - id is generated form this")
    value: str = Field(description="Value of the setting")
    
    @model_validator(mode='before')
    @classmethod
    def _f(cls, values):
        values['id'] = make_uuid({"key": values['key']})
        return values
    
    @classmethod
    def inserts_from_map(cls, values:dict):
        """given some key value settings, generate an insert batch statement - might be a good idea to wrap these in try blocks """
        if not values:
            return ""

        statements = []
        for key, val in values.items():
            # Convert value to JSON-safe string, then escape for SQL
            escaped_key = json.dumps(key)
            sql_key = f"'{key}'"
            escaped_val = f"'{val}'"

            """we use a database function that hashes the key for the id and then we add the key value pair"""
            stmt = f"""
INSERT INTO p8."Settings" (id, key, value)
SELECT p8.json_to_uuid('{{"key": {escaped_key}}}'::JSONB), {sql_key}, {escaped_val}
ON CONFLICT (id)
DO UPDATE SET value = EXCLUDED.value;
""".strip()
            statements.append(stmt)

        return "\n".join(statements)
  
# Scheduled tasks model
class Schedule(AbstractModel):
    """Defines a scheduled task.
    Tasks are scheduled by the system or by users and can be a system prompt for an agent or a function call
    """
    id: typing.Optional[uuid.UUID| str] = Field(None, description="Unique schedule id")
    userid: typing.Optional[uuid.UUID| str] = Field(None, description="User id associated with schedule")
    name: str = Field(..., description="Task to execute")
    spec: dict = Field(..., description="the task spec - The task spec can be any json but we have an internal protocol. LLM instructions are valid should use a system_prompt attribute")
    schedule: str = Field(..., description="Cron schedule string, e.g. '0 0 * * *'")
    disabled_at: typing.Optional[datetime.datetime] = Field(None, description="Time when schedule was disabled")

    @model_validator(mode='before')
    @classmethod
    def _set_defaults(cls, values):
        if isinstance(values, dict) and not values.get('id'):
            values['id'] = str(uuid.uuid1())
        return values
 
 ###
 ## TODO: the functions on this object will move to SQL function and API level 
 ###
class Engram(AbstractModel):
    """Your role is to observe user text and then create a user memory.
    A user memory is a link between a `User` node named <user> and a <relationship> and a named concept.
    You should choose from canonical relationship names if you can;
    
    - only use knows relationships for person knows person
    - feel free to skip relationships that are not defined in the list below
    
    Dont assume strong relationships e.g. if someone is 'doing' something that does not mean they are 'skilled' in it.
    You should look for strong evidence to support each relationship or not mention it at all.........................
        
    **Relationship: **
    - `knows`
    - `likes`
    - `likes-not` 
    - `has-allergy` 
    - `has-goal`
    - `has-skill`
    - `working-on`
    - `is-learning`
    - `recommends`
    - `has-value`
    - `fears`
    - `takes-medication-for` 
    - `plans-to-attend` 
    - `advocates-for`
    - `opposes`
    - `teaches`
    - `aspires-to`
    - `reports-to`
    - 'observed'
    """

    @classmethod
    def _add_memory_from_user_sessions(cls, since_days_ago:int=1,limit:int=200):
        """
        A helper method to read the data from the sessions and generate memories
        
        Args:
            since_days_ago: how far back in days to look for updated session records
            limit: primarily not to overload models, limit the records. 
            Users are unlikely to have more than 100 sessions but file uploads count as sessions
            TODO: note this is currently for all users so we need to think about scale
        """
        
        from percolate.utils import get_days_ago_iso_timestamp 
        dt = get_days_ago_iso_timestamp(n=since_days_ago)
        
        """NOTE: we need to harden these types"""
        #in this case this filters things that are not based on user comments
        session_type ='conversation'
        
        data = p8.repository(Session).execute(f""" SELECT a.query, a.created_at, a.userid, a.updated_at, email, u.name
                                        FROM p8."Session" a
                                         join p8."User" u on a.userid = u.id 
                                        where a.updated_at > %s 
                                        and session_type = %s
                                        order by a.updated_at desc 
                                        limit {limit}""", data=(dt,session_type))
        
        logger.debug(f"Fetched {len(data)} records to convert to memories")
        
        if not data:
            return
        
        return p8.Agent(Engram).run(f"""Please add memories for the users by their name
                                    - the user_name comes from the email in the data 
                                    - check the function signature for parameter structure
                                    ```json
                                    {json.dumps(data,default=str)}
                                    ```
                                    """)
            
    @classmethod
    def add_memory(cls, relationships: typing.List[dict]):
        """
        Add a user memory by linking a user to a concept by a relationships type
        You should determine the user name from the content
        Te relationships are a list of dicts with `user_name`, `relationship` and `target_name`
        For target names try to use simple keywords and phrases without special characters or punctuation
        
        Args:
            relationships: a list of relationships object|dicts with `user_name`, `relationship` and `target_name`
        """
        
        from percolate.services import PostgresService
        
        for r in relationships:
            r['source_label'] = 'User'
            r['source_name'] = r['user_name']
            r['rel_type'] = r['relationship'].replace('-','_')
            
        r = PostgresService().graph.add_relationships(relationships)
                
        return r
    
    @classmethod
    def describe_user(cls, user_name:str):
        """supply a user name and get their memory graph to construct a narrative for that person 
        - feel free to infer some things the person might do or like but be clear if its assumed
        Make the description conversational rather than bullet points.
        Args:
            user_name: the unique user name as given
        """
        from percolate.services import PostgresService
         
        return PostgresService().graph.get_user_concept_links(user_name,as_model=True)
        

class AuditStatus(Enum):
    """Status enum for audit"""
    Fail="Fail"
    Success="Success"
class Audit(AbstractModel):
    """Generic event audit for Percolate"""
    id: typing.Optional[uuid.UUID| str] = Field(None, description="Unique id")
    userid: typing.Optional[uuid.UUID| str] = Field(None, description="Optional user context")
    status: str = Field(..., description="audit status e.g. Fail|Success")
    caller:str = Field(..., description="The calling object e.g. a class or service")
    status_payload: typing.Optional[dict] = Field(default_factory={}, description="Details about the event")
    error_trace : typing.Optional[str] = Field(None, description="If there is an error dump the stack trace")


# """The daily digest agent will evolve with the help of some API utils
# - we want to summarize changes in the user including their readings etc
# - we want to summarize new resources they have uploaded 
# - we want to specifically look at their evolving memory graph
# """
class DigestAgent(AbstractModel):
    """You are the digestion agent for the user - generate daily or weekly digests to email to the user.
    - Users upload resources like images, document, audio etc and we process them every day
    - Users also interact via that sessions
    - As users interact we build a knowledge graph for their memories. 
    Every time period you should look at what is changing and give a user summary. Create sections. 
    Only add the section when there is something relevant and do not comment on the fact that there is nothing to add if there is not.
    1. Provide a brief summary of the user; their goals, fears, thoughts and dreams as though you are taking to them like an oracle or good friend
    2. Do not be to nice to them. Be polite but challenge them to be better and to grow
    3. Call out any interesting information related to their goals
    4. Call out any appointments or reminders that came up that day that they may want to recall
    5. When looking at graph edges, observe any new edges added based on timestamps
    6. Call out an logical or factual inaccuracies or any conflicts or contradictions that they may want to
    7. Suggest any tasks they may want to keep an eye to get closer to their goals and any new tasks they may want to initiate (you may need access to their full task list)
    8. If you have access to their calendar (disabled for now) list any appointments coming up in the following day or two
    """
    @classmethod
    def get_daily_digest(cls,name:str, user_name:str,**kwargs):
        """
        The daily digest will be a combination of data feeds which could be pushed into the data tier
        It should be resource uploaded in the last N hours and the users profile
        """
        from percolate.utils import get_days_ago_iso_timestamp
        
        #fetch by the user name which is the email address
        user = p8.repository(User).execute(""" SELECT * FROM p8."User" where email = %s """, data=(user_name,))
        if not user:
            raise Exception(f"Could not find user by name `{user_name}`")
        if isinstance(user,list):
            user = user[0]
        user_id = user['id']
        
        #TODO we will unify user and p8 user
        q= f""" 'MATCH p = (u:User {{name: ''{user_name}''}})-[r*1..2]->(c:Concept)
                   RETURN u.name,  c.name as concept, relationships(p) as relationships  ',
                    'name agtype,  concept agtype, relationships agtype'"""
                                                    
        user_graph = p8.repository(User).execute(f"""   select * from cypher_query({q})  """)
        #TODO compact the graph
        
        """if would be good to render the graph as an image and make it available
           the caller can select the user_graph and do that if they wish or the agent can just build a summary from it
        """
        
        recent_resources_summarized = p8.repository(Resources).execute(""" SELECT * FROM p8."Resources" where userid=%s and updated_at >= %s  """,
                                                                            data=(user_id, get_days_ago_iso_timestamp(n=1)))
        
        """do a check on the context length for now and summarize if there is too much"""
        
        return {
            'user_last_session': user['updated_at'],
            'digest_scope': name,
            'user': user,
            'user_graph': user_graph,
            'recent_resources_upload': recent_resources_summarized            
        }