import importlib
import json
import logging
import os
import sys
import traceback
import uuid
from contextlib import suppress
from copy import deepcopy
from datetime import datetime
from importlib import import_module
from typing import Any, Callable, Dict, Optional

import magic
from celery import chain, shared_task
from croniter import croniter
from django.apps import apps
from django.conf import settings
from django.contrib.contenttypes.fields import GenericForeignKey
from django.contrib.contenttypes.models import ContentType
from django.core.exceptions import ValidationError
from django.core.files.base import ContentFile
from django.db import models, transaction
from django.db.models import Q
from django.db.models.signals import post_delete, pre_delete
from django.db.utils import IntegrityError
from django.dispatch import receiver
from django.utils import timezone
from django.utils.functional import cached_property
from django.utils.translation import gettext_lazy as _
from django_celery_beat.models import CrontabSchedule, PeriodicTask, cronexp
from import_export.resources import Resource
from picklefield.fields import PickledObjectField
from tablib import Dataset
from tqdm import tqdm
from wbcore.contrib.notifications.dispatch import send_notification
from wbcore.contrib.notifications.utils import create_notification_type
from wbcore.utils.importlib import import_from_dotted_path
from wbcore.utils.models import ComplexToStringMixin

from .enums import ExportFormat, get_django_import_export_format
from .exceptions import ImportError

logger = logging.getLogger("io")


class ParserHandler(models.Model):
    class MimeTypeChoices(models.TextChoices):
        CSV = "text/csv", "CSV"
        JSON = "application/json", "JSON"
        XLS = "application/vnd.ms-excel", "XLS"
        XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", "XLSX"

    parser = models.CharField(max_length=255)
    handler = models.CharField(max_length=255)

    allow_file_type = models.CharField(max_length=68, blank=True, null=True, choices=MimeTypeChoices.choices)

    def __str__(self) -> str:
        return f"{self.parser}::{self.handler}"

    def parse(self, import_source: "ImportSource") -> Dict[str, Any]:
        """
        call the parser on the provided data
        Args:
            import_source: The import source containing the data
        Returns:
            The parsed data
        """

        parser = importlib.import_module(self.parser)
        return parser.parse(import_source)

    def handle(self, import_source: "ImportSource", parsed_data: Dict[str, Any], **kwargs):
        """
        Call the Handler on the parsed data
        Args:
            import_source: The initial import source
            parsed_data: The parsed_data
        """
        if parsed_data:
            model: Any = apps.get_model(self.handler)
            if handler_class := getattr(model, "import_export_handler_class", None):
                handler = handler_class(import_source)
                handler.process(parsed_data, **kwargs)

    class Meta:
        unique_together = ("parser", "handler")
        verbose_name = "Parser-Handler"
        verbose_name_plural = "Parsers-Handlers"

    @classmethod
    def get_representation_value_key(self) -> str:
        return "id"

    @classmethod
    def get_representation_label_key(self) -> str:
        return "{{parser}}::{{handler}}"

    @classmethod
    def get_representation_endpoint(self) -> str:
        return "wbcore:io:parserhandlerrepresentation-list"


class ImportedObjectProviderRelationship(ComplexToStringMixin, models.Model):
    """
    A model that represent the relationship/link between the imported object and a provider.

    This model can be used to define different identifier in case the object is imported from difference providers
    """

    provider = models.ForeignKey(
        to="io.Provider", related_name="imported_object_relationships", on_delete=models.CASCADE
    )
    content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE)
    object_id = models.PositiveIntegerField()
    content_object = GenericForeignKey("content_type", "object_id")
    provider_identifier = models.CharField(max_length=255)

    def compute_str(self) -> str:
        try:
            return f"{self.provider.title} -> {self.content_object}: {self.provider_identifier}"
        except AttributeError:
            return f"{self.provider.title} ({self.provider_identifier})"

    class Meta:
        verbose_name = "Content object Provider Identifier relationship"
        verbose_name_plural = "Content object Provider Identifier relationships"
        unique_together = [
            ("content_type", "object_id", "provider"),
            ("content_type", "provider_identifier", "provider"),
        ]
        indexes = [
            models.Index(fields=["content_type", "object_id", "provider"]),
            models.Index(fields=["content_type", "provider_identifier", "provider"]),
        ]


class Provider(models.Model):
    """
    Represent a data vendor/provider. Every source is linked to a provider and are represented by a unique identifier
    extracted from the data backend root folder

    Example:
        io/backend/refinitiv/instrument_prices.py => Provider.objects.get(key="refinitiv")
    """

    title = models.CharField(max_length=255)
    key = models.CharField(max_length=255, unique=True)

    def __str__(self) -> str:
        return f"{self.title} ({self.key})"

    class Meta:
        verbose_name = "Provider"
        verbose_name_plural = "Providers"


class DataBackend(models.Model):
    """
    Represents the instantiated backend imported through the specified dotted path and the passed parameters
    """

    title = models.CharField(max_length=255)

    save_data_in_import_source = models.BooleanField(
        default=True, help_text="If true, save the data in the import_source json field"
    )
    passive_only = models.BooleanField(
        default=True, help_text="If True, this data backend is allowed to be called only from the import source."
    )

    backend_class_path = models.CharField(max_length=512)
    backend_class_name = models.CharField(max_length=128, default="DataBackend")

    provider = models.ForeignKey(
        "io.Provider",
        on_delete=models.SET_NULL,
        null=True,
        blank=True,
        verbose_name=_("Provider"),
    )

    def get_internal_object(self, object_content_type: ContentType, provider_external_id: str) -> models.Model | None:
        """
        Internal facing function that returns the internal id of the object with the given external provider identifier. If the relationship already exists, we simply returns the stored id. Otherwise we call the appropriate function to get
        Args:
            object_content_type: ContentType object representing the seeked object id
            provider_external_id: The object identifier as given by this backend provider

        Returns:
            The object as stored in the importer object-provider relationship. None if it doesn't exist yet
        """
        if rel := ImportedObjectProviderRelationship.objects.filter(
            provider=self.provider, provider_identifier=provider_external_id, content_type=object_content_type
        ).first():
            return rel.content_object

    class Meta:
        verbose_name = "Data Backend"
        verbose_name_plural = "Data Backends"

    @cached_property
    def backend_class(self) -> Any:
        """
        Return the imported backend class
        Returns:
            The backend class
        """
        return getattr(import_module(self.backend_class_path), self.backend_class_name)

    def __str__(self) -> str:
        return self.title

    @classmethod
    def get_representation_value_key(self) -> str:
        return "id"

    @classmethod
    def get_representation_label_key(self) -> str:
        return "{{title}}"

    @classmethod
    def get_representation_endpoint(self) -> str:
        return "wbcore:io:databackendrepresentation-list"


class Source(models.Model):
    title = models.CharField(max_length=128)
    uuid = models.UUIDField(default=uuid.uuid4, editable=False, unique=True)
    parser_handler = models.ManyToManyField(
        "io.ParserHandler",
        verbose_name="Parser/Handler",
        related_name="sources",
    )

    is_active = models.BooleanField(default=True)

    connection_parameters = models.JSONField(default=dict, null=True, blank=True)
    import_parameters = models.JSONField(default=dict, null=True, blank=True)

    credentials = models.ManyToManyField("io.ImportCredential", blank=True, related_name="sources")

    data_backend = models.ForeignKey("io.DataBackend", on_delete=models.CASCADE, verbose_name=_("Data Backend"))

    crontab = models.ForeignKey(
        CrontabSchedule,
        on_delete=models.SET_NULL,
        null=True,
        blank=True,
        verbose_name=_("Crontab Schedule"),
        help_text=_("Crontab Schedule to run the task on.  " "Set only one schedule type, leave the others null."),
    )

    periodic_task = models.ForeignKey(
        PeriodicTask,
        on_delete=models.SET_NULL,
        null=True,
        blank=True,
        verbose_name="Periodic Task",
        related_name="sources",
    )

    import_timedelta_interval = models.IntegerField(default=0)

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.instantiated_backend: Any = None

    def init_backend(self, execution_datetime: datetime):
        """
        Singleton that instantiate the backend attribute

        Args:
            execution_datetime: Datetime at which the backend is supposed to be initialize

        """
        if not self.instantiated_backend:
            self.instantiated_backend = self.data_backend.backend_class(
                import_credential=self.get_valid_credential(execution_datetime),
                data_backend=self.data_backend,
                **self.connection_parameters,
            )

    def save(self, *args, **kwargs):
        super().save(*args, **kwargs)
        self.update_periodic_task()

    def __str__(self) -> str:
        return self.title if self.title else f"{self.uuid}"

    class Meta:
        verbose_name = "Source"
        verbose_name_plural = "Sources"

    @property
    def current_valid_credential(self) -> "ImportCredential":
        """
        Property that returns the current valid credential

        Returns:
            The valid ImportCredential (as of now)
        """
        return self.get_valid_credential(timezone.now())

    @property
    def crontab_repr(self) -> str:
        """
        Returns the crontab string representation.
        :return: crontab string representation
        """
        if not self.crontab:
            return ""

        return "{0} {1} {2} {3} {4}".format(
            cronexp(self.crontab.minute),
            cronexp(self.crontab.hour),
            cronexp(self.crontab.day_of_month),
            cronexp(self.crontab.month_of_year),
            cronexp(self.crontab.day_of_week),
        )

    def update_periodic_task(self):
        """
        Utility function to find and update the link periodic task given the current source state
        """
        if self.crontab:
            if task := self.periodic_task:
                task.crontab = self.crontab
                task.enabled = self.is_active
                task.save()
            else:
                task = PeriodicTask.objects.update_or_create(
                    name=f"Import-Export: {self}",
                    defaults={
                        "crontab": self.crontab,
                        "args": json.dumps([self.pk]),
                        "task": "wbcore.contrib.io.models.trigger_workflow_as_task",
                    },
                )[0]
            Source.objects.filter(pk=self.pk).update(periodic_task=task)
            self.refresh_from_db()

    def get_valid_credential(self, val_datetime: datetime) -> "ImportCredential":
        """
        Get the valid credential
        Args:
            val_datetime: The date at which the credentials need to be valid
        Returns:
            A valid credential
        """
        return (
            self.credentials.filter(
                (Q(validity_start__lte=val_datetime) | Q(validity_start__isnull=True))
                & (Q(validity_end__gte=val_datetime) | Q(validity_end__isnull=True))
            )
            .order_by("-validity_end")
            .first()
        )

    def get_or_create_provider_id(self, obj: models.Model) -> str | None:
        """
        Workflow method that try to get or fetch the provider identifier for the passed object.

        If the relationship already exists, we directly returns the stored external id. Otherwise we call the appropriate function on the already instantiated backend to get it from the provider itself. If the value is returned, we store the relationship

        Args:
            obj: The object to get the external provider identifier from

        Returns:
            If it exists, the external identifier (from this source provider). None otherwise.
        """

        if (provider := self.data_backend.provider) and self.instantiated_backend is not None:
            if self.instantiated_backend.is_object_valid(obj):
                if rel := ImportedObjectProviderRelationship.objects.filter(
                    provider=provider, content_type=ContentType.objects.get_for_model(obj), object_id=obj.pk
                ).first():
                    return rel.provider_identifier
                else:
                    if provider_identifier := self.instantiated_backend.get_provider_id(obj):
                        with transaction.atomic():
                            ImportedObjectProviderRelationship.objects.create(
                                provider=provider,
                                content_type=ContentType.objects.get_for_model(obj),
                                object_id=obj.pk,
                                provider_identifier=provider_identifier,
                            )
                            return provider_identifier
            obj.save()
        else:
            raise ValueError(
                f"You can't create a provider relationship with {obj} with for backend {self.data_backend} without a provider assigned"
            )

    def is_valid_date(self, sync_datetime: datetime) -> bool:
        """
        check wether a date is valid given the stored crontab schedule
        Args:
            sync_datetime: The datetime at which validity needs to be checked
        Returns:
            True if the given date is valid given the source crontab
        """
        if not self.crontab:
            return False
        return croniter.match(self.crontab_repr, sync_datetime)

    def trigger_workflow(
        self,
        execution_time: Optional[datetime] = None,
        callback_signature: Callable | None = None,
        callback_args: list | None = None,
        callback_kwargs: Dict | None = None,
        synchronous: bool = False,
        **kwargs,
    ):
        """
        The entry point function of the whole import source workflow.
        Loop over all files return by the attached backend and save the result into a serie of import sources.

        Args:
            execution_time: The time at which the worfklow was triggerd
            callback_signature: The celery canvas signature to be called upon completion (if any)
            callback_args: the potential positional arguments to be passed to the callback function (if any)
            callback_kwargs: Potential keyword arguments to be passed to the callback function (if any)
            synchronous: If true, will execute the workflow synchronously. Default to False
            **kwargs: keyword arguments to be passed down along the workflow

        Returns:
            A generator of import sources
        """
        if not execution_time:
            execution_time = timezone.now()
        # If a queryset is provided, we need to convert it into a list of valid external id that we will get from the relationship through model or from the backend directly if not existing

        chained_tasks = [
            generate_import_sources_as_task.s(self.pk, execution_time, **kwargs),
            process_import_sources_as_task.s(),
        ]
        if not callback_args:
            callback_args = []
        if not callback_kwargs:
            callback_kwargs = {}
        if callback_signature:
            chained_tasks.append(callback_signature.si(*callback_args, **callback_kwargs))
        if synchronous:
            import_source_ids = chained_tasks[0].apply().get()
            chained_tasks[1].apply((import_source_ids,))
            if len(chained_tasks) > 2:
                chained_tasks[2].apply(tuple(callback_args), callback_kwargs)
        else:
            chain(*chained_tasks).apply_async(eta=execution_time)

    def generate_import_sources(
        self,
        execution_time: datetime,
        only_handler: str | None = None,
        **kwargs,
    ) -> list["ImportSource"]:
        """
                Process the source based on its attached backend (returns an error if such backend does not exist)
        The backend is first initialize through constructor and then, The generator `get_files` is called, passing the
        execution time, the expected data time, and some keyword arguments.

        For each file returns by the generator, we create a ImportSource objects and trigger its import task
        Args:
            execution_time: The time at which this task was triggered
            **kwargs: keyword arguments to be passed down along the workflow
        Returns:
            Return a generator of import sources
        """
        if not self.data_backend:
            raise ValueError("Data Backend not specied for this source")

        # Import the data backend module and extract its class_name
        res = []
        # Get the expected date at which this source should return its data (e.g. T-1)

        # Loop over the backend returned generator
        self.init_backend(execution_time)
        errors = []
        queryset = kwargs.pop("queryset", self.instantiated_backend.get_default_queryset())
        if self.data_backend.provider and queryset is not None:
            obj_external_ids = []
            for obj in tqdm(queryset, total=queryset.count()):
                try:
                    if external_id := self.get_or_create_provider_id(obj):
                        obj_external_ids.append(external_id)
                except IntegrityError:
                    errors.append(str(obj))
            kwargs["obj_external_ids"] = obj_external_ids
        else:
            kwargs["queryset"] = queryset
        streams = self.instantiated_backend.get_files(
            execution_time,
            **{
                **self.import_parameters,
                **kwargs,
            },
        )
        if streams:
            for file_name, _file in streams:
                content_file = ContentFile(_file.getvalue())
                content_file.name = file_name
                parser_handlers = self.parser_handler.all()
                if only_handler:
                    parser_handlers = parser_handlers.filter(handler__iexact=only_handler)
                for parser_handler in parser_handlers:
                    with transaction.atomic():
                        import_source = ImportSource.objects.create(
                            file=content_file,
                            parser_handler=parser_handler,
                            save_data=self.data_backend.save_data_in_import_source,
                            source=self,
                        )
                        res.append(import_source)
        return res

    @classmethod
    def load_sources_from_settings(self, settings: list[tuple[list[tuple[str, str]], str, dict[str, Any]]]):
        """
        Utility classmethod to parser sources from the settings.

        We assume data structure as follow

        [
            (
                [("module1.HandlerModel1", "module1.io.parsers.provider1.handler1")],
                "module1.io.backends.provider1.handler1.DataBackend",
                {
                    "credentials": [credential_dict],
                    "crontab": "10 9,17,1 * * *",
                },
            ),
            (
                [("module2.HandlerModel2", "module2.io.parsers.provider2.handler2")],
                "module2.io.backends.provider2.handler2.DataBackend",
            )
        ]
        Args:
            settings: The source settings to parse

        """
        for parser_handlers, data_backend_module, config in settings:
            with suppress(Exception):
                # This method is loaded when server boots. If this fail, it can prevent the server from starting.
                backend_class_path, _, backend_class_name = data_backend_module.rpartition(".")

                data_backend = DataBackend.objects.get(
                    backend_class_path=backend_class_path, backend_class_name=backend_class_name
                )
                crontab = None
                credentials = config.pop("credentials", [])
                if crontab_data := config.pop("crontab", None):
                    minute, hour, day_of_month, month_of_year, day_of_week = crontab_data.split(" ")
                    crontab_kwargs = {
                        "minute": minute,
                        "hour": hour,
                        "day_of_week": day_of_week,
                        "day_of_month": day_of_month,
                        "month_of_year": month_of_year,
                    }
                    crontabs = CrontabSchedule.objects.filter(**crontab_kwargs)
                    if crontabs.exists():
                        crontab = crontabs.first()
                    else:
                        crontab = CrontabSchedule.objects.create(**crontab_kwargs)

                source, source_created = Source.objects.get_or_create(
                    data_backend=data_backend,
                    defaults={"crontab": crontab, **config},
                )
                for handler, parser in parser_handlers:
                    parser_handler = ParserHandler.objects.get_or_create(parser=parser, handler=handler)[0]
                    if parser_handler not in source.parser_handler.all():
                        source.parser_handler.add(parser_handler)
                for credential_data in credentials:
                    credential, created = ImportCredential.objects.get_or_create(
                        key=credential_data["key"], defaults=credential_data
                    )
                    if credential not in source.credentials.all():
                        source.credentials.add(credential)
                source.save()

    @classmethod
    def get_representation_value_key(self) -> str:
        return "id"

    @classmethod
    def get_representation_label_key(self) -> str:
        return "{{title}} ({{id}})"

    @classmethod
    def get_representation_endpoint(self) -> str:
        return "wbcore:io:sourcerepresentation-list"


@receiver(post_delete, sender="io.Source")
def post_delete_source(sender, instance, **kwargs):
    # Delete the attached period task
    if instance.periodic_task:
        instance.periodic_task.delete()


@receiver(pre_delete, sender="io.Source")
def pre_delete_source(sender, instance, **kwargs):
    # Unassign import source from the deleting source
    ImportSource.objects.filter(source=instance).update(source=None)


@shared_task(queue="importexport")
def trigger_workflow_as_task(source_id: int, execution_time: datetime | None = None, **kwargs):
    """
    Call the `import_source` as a celery task
    """
    source = Source.objects.get(id=source_id)
    source.trigger_workflow(execution_time=execution_time, **kwargs)


@shared_task(queue="importexport")
def generate_import_sources_as_task(source_id: int, execution_time: datetime, **kwargs) -> list[int]:
    """
    Call the `import_source` as a celery task
    """
    source = Source.objects.get(id=source_id)
    import_source_ids = [
        import_source.pk for import_source in source.generate_import_sources(execution_time, **kwargs)
    ]  # convert to list to be json serializable
    return import_source_ids


@shared_task(queue="importexport")
def process_import_sources_as_task(import_source_ids: list[int]):
    """
    Call the `import_source` as a celery task
    """
    errors = []
    for import_source_id in import_source_ids:
        import_source = ImportSource.objects.get(id=import_source_id)
        try:
            import_source.import_data(force_reimport=True)
        except ImportError:
            errors.append(import_source_id)
    if len(errors) > 0:
        raise ImportError(f"Error while processing import source {errors}")


class ImportExportSource(models.Model):
    class Status(models.TextChoices):
        PENDING = "PENDING", "Pending"
        PROCESSED = "PROCESSED", "Processed"
        WARNING = "WARNING", "Warning"
        ERROR = "ERROR", "Error"
        IGNORED = "IGNORED", "Ignore"

    created = models.DateTimeField(auto_now_add=True)
    last_updated = models.DateTimeField(auto_now=True)

    status = models.CharField(
        max_length=16,
        choices=Status.choices,
        default=Status.PENDING.value,
    )

    log = models.TextField(null=True, blank=True)
    origin = models.CharField(max_length=255, null=True, blank=True)
    creator = models.ForeignKey(
        "authentication.User", null=True, blank=True, verbose_name="Creator", on_delete=models.SET_NULL
    )
    data = models.JSONField(default=dict, null=True, blank=True)

    class Meta:
        abstract = True


class ExportSource(ImportExportSource):
    file = models.FileField(max_length=256, upload_to="io/export_source/files", null=True, blank=True)
    format = models.IntegerField(
        choices=ExportFormat.choices,
        default=ExportFormat.CSV.value,
        verbose_name=_("Format of file to be exported"),
    )

    content_type = models.ForeignKey(ContentType, verbose_name=_("Export job Content Type"), on_delete=models.CASCADE)

    resource_path = models.CharField(
        verbose_name=_("Resource path to use when exporting"), max_length=255, default="", blank=True, null=True
    )

    resource_kwargs = models.JSONField(default=dict, blank=True, null=False)

    query_str = models.TextField(verbose_name=_("SQL query to be executed"), blank=True, null=False)
    query_params = PickledObjectField(  # we have to use picklefield because the sql parameters are given as python object. However, no django model should be stored in this field, which should be the case as these objects are given through their IDs
        blank=True, verbose_name=_("SQL query parameters to be used with the sql query"), default=list
    )

    class Meta:
        verbose_name = _("Export Source")
        verbose_name_plural = _("Export Sources")
        notification_types = [
            create_notification_type(
                code="io.export_done",
                title="File Export has finished",
                help_text="Notifies user when their submitted export files is done and available",
                web=True,
                mobile=True,
                email=True,
            )
        ]
        constraints = [
            models.CheckConstraint(
                check=Q(query_str__isnull=False, resource_path__isnull=False) | ~Q(data__exact=dict()),
                name="check_either_data_or_resource_isnotnull",
            )
        ]

    @property
    def file_format(self):
        return get_django_import_export_format(self.format)()

    @property
    def resource(self) -> Resource:
        """
        Load into an attribute the instantiated resource loaded from the resource path
        """
        resource_class = import_from_dotted_path(self.resource_path)
        return resource_class(**self.resource_kwargs)

    @property
    def queryset(self) -> models.QuerySet:
        """
        Recreate the base queryset from the content type model class base manager and the saved query string and parameters
        """

        return self.content_type.model_class().objects.raw(self.query_str, self.query_params)

    def get_export_filename(self):
        date_str = timezone.now().strftime("%Y-%m-%d")
        ts = datetime.timestamp(timezone.now())
        filename = "%s-%s_%s.%s" % (
            self.content_type.model_class().__name__,
            date_str,
            ts,
            self.file_format.get_extension(),
        )
        return filename

    def notify(self):
        """
        Notify the user if the export job is done and accessible
        """
        if self.file and self.status == self.Status.PROCESSED:
            send_notification(
                code="io.export_done",
                title=_("Your export file is available"),
                body=_("<p>The export job you requested is finished and available for one hour.</p>").format(
                    url=self.file.url
                ),
                endpoint=self.file.url,
                user=self.creator,
            )

    def export_data(self, **kwargs):
        self.log = ""
        self.status = self.Status.PENDING.name
        self.save()
        try:
            file_format = get_django_import_export_format(self.format)()
            if self.data:
                # Data was saved as a dictionary from a tablib.dataset, we need to recreate it
                data = Dataset()
                data.headers = self.data["headers"]
                data.extend(self.data["data"])
            else:
                data = self.resource.export(queryset=self.queryset)

            export_data = file_format.export_data(data)
            if not file_format.is_binary():
                export_data = export_data.encode("utf-8")
            self.file.save(self.get_export_filename(), ContentFile(export_data))
            self.status = self.Status.PROCESSED.name
            self.save()
            self.notify()

        except Exception:
            ex_type, ex_value, _ = sys.exc_info()
            self.status = self.Status.ERROR.name
            self.log = f"{ex_type}: {ex_value}\n"
            self.log += traceback.format_exc()
            self.save()


class ImportSource(ImportExportSource):
    file = models.FileField(max_length=256, upload_to="io/import_source/files", null=True, blank=True)

    save_data = models.BooleanField(default=True, help_text="If True, will save the raw data in a field")
    progress_index = models.PositiveIntegerField(default=0)

    parser_handler = models.ForeignKey("io.ParserHandler", on_delete=models.CASCADE)
    source = models.ForeignKey("io.Source", null=True, blank=True, verbose_name="Source", on_delete=models.SET_NULL)
    errors_log = models.TextField(null=True, blank=True)

    class Meta:
        verbose_name = "Import Source"
        verbose_name_plural = "Import Sources"
        notification_types = [
            create_notification_type(
                code="io.import_done",
                title="File Import has finished",
                help_text="Notifies user when their submitted import files has finished",
            )
        ]

    def __str__(self) -> str:
        return f"{self.pk}"

    def _validate_file_type(self):
        """
        Valid the file type based on the allowed types specified by the parser
        Returns:
            True if the filetype is valid
        """
        if (upload := self.file) and (allow_file_type := self.parser_handler.allow_file_type):
            file_type = magic.from_buffer(upload.file.read(1024), mime=True)
            if file_type != allow_file_type:
                raise ValidationError("File type not supported.")

    def save(self, *args, **kwargs):
        # Check here company name, modify ...
        self._validate_file_type()
        super().save(*args, **kwargs)

    def _parse_data(self) -> dict[str, Any]:
        """
        Given the ParserHandler linked object, we import its related module and call its defined `parse` function
        Returns:
            Dict The parsed data as a python dictionary
        """
        if not self.file:
            raise ValueError("This import source does not include a valid file")
        parsed_data = self.parser_handler.parse(self)
        if self.save_data:
            self.data = parsed_data
        self.save()
        return parsed_data

    def _process_data(self, parsed_data: dict[str, Any], **kwargs):
        """
        Given the parsed data (as a python dictionary), import the corresponding handler and calls its defined `process
        Args:
            data: The deserialized data to be passed down to the parser
        """
        self.status = self.Status.PROCESSED.name
        if parsed_data:
            parsed_data_copy = deepcopy(parsed_data)
            if data := parsed_data_copy.pop("data", None):
                self.parser_handler.handle(self, {"data": data[self.progress_index :], **parsed_data_copy}, **kwargs)
                if self.errors_log:
                    self.status = self.Status.WARNING.name
        self.save()

    def import_data(self, force_reimport: Optional[bool] = False, **kwargs):
        """
        General workflow method:
        * Parse the data given the linked import file
        * Handle the parsed data
        * Change the import_source status based on sucess
        Args:
            silent: False if this method is not suppose to silently catch the exceptions
        """
        if force_reimport:
            self.progress_index = 0
        self.log = ""
        self.errors_log = ""
        # self.data = {}
        self.status = self.Status.PENDING.name
        self.save()
        debug = kwargs.get("debug", settings.DEBUG)
        try:
            data = self._parse_data()
            self._process_data(data, debug=debug)
        except Exception as e:
            ex_type, ex_value, _ = sys.exc_info()
            self.status = self.Status.ERROR.name
            self.log = f"{ex_type}: {ex_value}\n"
            self.log += traceback.format_exc()
            self.save()
            if not debug:
                raise ImportError(
                    f"Could not import file {self.file.name} for backend {self.source.data_backend.title} and parser/handler {self.parser_handler}"
                )
            else:
                raise e

    @classmethod
    def get_representation_value_key(self) -> str:
        return "id"

    @classmethod
    def get_representation_label_key(self) -> str:
        return "{{file}}"

    @classmethod
    def get_representation_endpoint(self) -> str:
        return "wbcore:io:importsourcerepresentation-list"


@shared_task(queue="importexport")
def import_data_as_task(import_source_id: int, **kwargs):
    """
    Call `import_data` as a celery task
    :param Int The import_source id
    """
    import_source = ImportSource.objects.get(id=import_source_id)
    import_source.import_data(**kwargs)


@shared_task(queue="importexport")
def export_data_as_task(export_source_id: int, **kwargs):
    """
    Call `import_data` as a celery task
    :param Int The import_source id
    """
    export_source = ExportSource.objects.get(id=export_source_id)
    export_source.export_data(**kwargs)


@receiver(models.signals.post_save, sender=ExportSource)
def post_save_export_source(sender, instance: ExportSource, created: bool, raw: bool, **kwargs):
    """Triggers the export task on creation"""

    if not raw and created and instance.status == ExportSource.Status.PENDING:
        transaction.on_commit(lambda: export_data_as_task.delay(instance.id))


def validate_key_file(value: models.FileField):
    ext = os.path.splitext(value.name)[1]
    if ext.lower() != ".key":
        raise ValidationError("The file extension needs to be a .key")


def validate_pem_file(value: models.FileField):
    ext = os.path.splitext(value.name)[1]
    if ext.lower() != ".pem":
        raise ValidationError("The file extension needs to be a .pem")


class ImportCredential(models.Model):
    class Type(models.TextChoices):
        CREDENTIAL = "CREDENTIAL", "Credential"
        AUTHENTICATION_TOKEN = "AUTHENTICATION_TOKEN", "Authentication Token"
        CERTIFICATE = "CERTIFICATE", "Certificate"

    key = models.CharField(max_length=255)
    type = models.CharField(choices=Type.choices, default=Type.CREDENTIAL, max_length=64)

    username = models.CharField(max_length=255, null=True, blank=True)
    password = models.CharField(max_length=255, null=True, blank=True)

    authentication_token = models.CharField(max_length=2048, null=True, blank=True)

    certificate_pem = models.FileField(
        max_length=256,
        null=True,
        blank=True,
        upload_to="io/import_credential/certificates",
        help_text="We are expecting a .cert file",
        validators=[validate_pem_file],
    )
    certificate_key = models.FileField(
        max_length=256,
        null=True,
        blank=True,
        upload_to="io/import_credential/certificates",
        help_text="We are expecting a .key file",
        validators=[validate_key_file],
    )

    additional_resources = models.JSONField(default=dict, null=True, blank=True)

    validity_start = models.DateTimeField(null=True, blank=True)
    validity_end = models.DateTimeField(null=True, blank=True)

    def save(self, *args, **kwargs):
        if self.type == self.Type.CREDENTIAL and (not self.username or not self.password):
            raise ValidationError("The type is credential, you need to specify a username and a password")
        elif self.type == self.Type.AUTHENTICATION_TOKEN and not self.authentication_token:
            raise ValidationError("The type is authentication header, you need to specify a valid header")
        elif self.type == self.Type.CERTIFICATE and (not self.certificate_key or not self.certificate_pem):
            raise ValidationError("The type is certificate, you need to specify valid public and private key files")
        super().save(*args, **kwargs)

    class Meta:
        verbose_name = "Import Credential"
        verbose_name_plural = "Import Credentials"

    def __str__(self) -> str:
        dates_repr = ""
        if self.validity_start:
            dates_repr += f"{self.validity_start:%Y-%m-%d %H:%M:%S}"
        if self.validity_end:
            dates_repr += f"- {self.validity_end:%Y-%m-%d %H:%M:%S}"
        if dates_repr:
            return f"{self.key} ({dates_repr})"
        return self.key
