import os
import json
import logging
import subprocess
import shutil
import time
import concurrent.futures
from typing import Dict, List, Optional, Tuple, Any
from datetime import datetime
from models import Device, Patch, DirectoryStructureResponse

# Get logger
logger = logging.getLogger(__name__)

class DeviceManager:
    """Handles scanning and managing device data"""

    def __init__(self, devices_folder="midi-presets/devices", sync_enabled=True):
        """
        Initialize the device manager with the path to the devices folder

        Args:
            devices_folder: Path to the devices folder
            sync_enabled: Whether to sync with the remote repository
        """
        self.devices_folder = devices_folder
        self.devices = {}  # Map of device name to device data
        self.manufacturers = []  # List of manufacturer names
        self.device_structure = {}  # Map of manufacturer to list of devices
        self._json_cache = {}  # Cache for loaded JSON files
        self._cache_timeout = 3600  # Cache timeout in seconds (1 hour)
        self.sync_enabled = sync_enabled

        # Validate that the midi-presets submodule exists and is up to date
        self._validate_midi_presets_submodule()

    def _validate_midi_presets_submodule(self):
        """
        Validate that the midi-presets git submodule exists and is up to date.
        If not, attempt to initialize it using git_operations.
        """
        import os
        from git_operations import git_sync

        logger.info("Validating midi-presets git submodule")

        # Skip sync if disabled
        if not self.sync_enabled:
            logger.info("Sync is disabled, skipping git submodule validation")
            # Still check if the directory exists
            midi_presets_dir = os.path.join(os.getcwd(), "midi-presets")
            if not os.path.exists(midi_presets_dir):
                logger.warning("midi-presets directory does not exist and sync is disabled")
            return

        # Check if the midi-presets directory exists
        midi_presets_dir = os.path.join(os.getcwd(), "midi-presets")
        if not os.path.exists(midi_presets_dir):
            logger.warning("midi-presets directory does not exist, attempting to initialize it")
            success, message, _ = git_sync()
            if not success:
                logger.error(f"Failed to initialize midi-presets submodule: {message}")
                return
            logger.info(f"Successfully initialized midi-presets submodule: {message}")
        else:
            # Check if it's a valid git repository
            git_dir = os.path.join(midi_presets_dir, ".git")
            if not (os.path.exists(git_dir) or os.path.exists(os.path.join(midi_presets_dir, "..", ".git", "modules", "midi-presets"))):
                logger.warning("midi-presets directory exists but is not a valid git repository, attempting to reinitialize it")
                success, message, _ = git_sync()
                if not success:
                    logger.error(f"Failed to reinitialize midi-presets submodule: {message}")
                    return
                logger.info(f"Successfully reinitialized midi-presets submodule: {message}")
            else:
                logger.info("midi-presets git submodule exists and appears to be valid")

    def _load_json_file(self, file_path: str, cache_timeout: int = None) -> Dict:
        """
        Load a JSON file with caching to improve performance

        Args:
            file_path: Path to the JSON file
            cache_timeout: Optional cache timeout in seconds (defaults to self._cache_timeout)

        Returns:
            Dictionary containing the JSON data
        """
        # Use instance cache_timeout if not provided
        if cache_timeout is None:
            cache_timeout = self._cache_timeout

        # Check if file exists
        if not os.path.exists(file_path):
            logger.warning(f"JSON file not found: {file_path}")
            return {}

        # Check if file is in cache and not expired
        if file_path in self._json_cache:
            timestamp, data = self._json_cache[file_path]
            if time.time() - timestamp < cache_timeout:
                logger.debug(f"Using cached JSON data for {file_path}")
                return data

        # Load file and update cache
        try:
            start_time = time.time()
            with open(file_path, 'r') as f:
                data = json.load(f)
            load_time = time.time() - start_time
            logger.debug(f"Loaded JSON file {file_path} in {load_time:.4f} seconds")

            # Update cache with the new timeout
            self._json_cache[file_path] = (time.time(), data)
            return data
        except json.JSONDecodeError as e:
            logger.error(f"Invalid JSON in file {file_path}: {str(e)}")
            return {}
        except Exception as e:
            logger.error(f"Error loading JSON file {file_path}: {str(e)}")
            return {}

    def run_git_sync(self) -> Tuple[bool, str]:
        """
        Run git submodule sync to update the midi-presets submodule
        Returns a tuple of (success, message)
        """
        logger.info("Running git submodule sync using git_operations module")

        # Skip sync if disabled
        if not self.sync_enabled:
            logger.info("Sync is disabled, skipping git submodule sync")
            return False, "Sync is disabled"

        from git_operations import git_sync

        # Use the more robust git_sync function from git_operations
        success, message, _ = git_sync()

        if success:
            logger.info(f"Git submodule sync completed successfully: {message}")
        else:
            logger.error(f"Git submodule sync failed: {message}")

        return success, message

    def _process_manufacturer(self, manufacturer: str) -> Tuple[Dict[str, Dict], List[str]]:
        """
        Process a single manufacturer directory

        Args:
            manufacturer: Name of the manufacturer

        Returns:
            Tuple of (devices, device_structure)
        """
        manufacturer_path = os.path.join(self.devices_folder, manufacturer)
        logger.debug(f"Processing manufacturer directory: {manufacturer}")

        # Initialize return values
        manufacturer_devices = {}
        manufacturer_device_structure = []

        try:
            # Get list of files and directories in manufacturer directory
            items = os.listdir(manufacturer_path)

            # Process device directories and JSON files
            device_dirs = [d for d in items if os.path.isdir(os.path.join(manufacturer_path, d)) and d != 'community']
            json_files = [f for f in items if f.endswith('.json')]

            logger.debug(f"Found {len(device_dirs)} device directories and {len(json_files)} JSON files in {manufacturer} directory")

            # Use a thread pool to process JSON files in parallel
            with concurrent.futures.ThreadPoolExecutor(max_workers=min(32, os.cpu_count() * 2)) as executor:
                # Submit tasks for each JSON file
                future_to_file = {
                    executor.submit(self._load_json_file, os.path.join(manufacturer_path, filename)): filename
                    for filename in json_files
                }

                # Process results as they complete
                for future in concurrent.futures.as_completed(future_to_file):
                    filename = future_to_file[future]
                    try:
                        device_data = future.result()

                        # Check if device has device_info with a name
                        if device_data and 'device_info' in device_data and 'name' in device_data['device_info']:
                            device_name = device_data['device_info']['name']

                            # Add manufacturer information
                            device_data['manufacturer'] = manufacturer

                            # Check for community folders
                            community_path = os.path.join(manufacturer_path, 'community')
                            community_folders = []

                            if os.path.exists(community_path) and os.path.isdir(community_path):
                                community_items = [f for f in os.listdir(community_path) 
                                                 if f.endswith('.json')]
                                community_folders = [os.path.splitext(f)[0] for f in community_items]
                                logger.debug(f"Found {len(community_folders)} community folders for {device_name}")

                            device_data['community_folders'] = community_folders

                            # Store the device data
                            manufacturer_devices[device_name] = device_data
                            manufacturer_device_structure.append(device_name)

                            logger.debug(f"Loaded device: {device_name} from manufacturer {manufacturer}")
                        else:
                            logger.warning(f"Device file '{filename}' does not have a device_info.name field")
                    except Exception as e:
                        logger.error(f"Error processing JSON file {filename}: {str(e)}")

            # Process device directories
            for device_dir in device_dirs:
                device_path = os.path.join(manufacturer_path, device_dir)
                logger.debug(f"Processing device directory: {device_dir}")

                # Get JSON files in device directory
                device_json_files = [f for f in os.listdir(device_path) if f.endswith('.json')]

                if device_json_files:
                    # Use a thread pool to process JSON files in parallel
                    with concurrent.futures.ThreadPoolExecutor(max_workers=min(32, os.cpu_count() * 2)) as executor:
                        # Submit tasks for each JSON file
                        future_to_file = {
                            executor.submit(self._load_json_file, os.path.join(device_path, filename)): filename
                            for filename in device_json_files
                        }

                        # Process results as they complete
                        for future in concurrent.futures.as_completed(future_to_file):
                            filename = future_to_file[future]
                            try:
                                device_data = future.result()

                                # Check if device has device_info with a name
                                if device_data and 'device_info' in device_data and 'name' in device_data['device_info']:
                                    device_name = device_data['device_info']['name']

                                    # Add manufacturer information
                                    device_data['manufacturer'] = manufacturer

                                    # Check for community folders
                                    community_path = os.path.join(manufacturer_path, 'community')
                                    community_folders = []

                                    if os.path.exists(community_path) and os.path.isdir(community_path):
                                        community_items = [f for f in os.listdir(community_path) 
                                                         if f.endswith('.json')]
                                        community_folders = [os.path.splitext(f)[0] for f in community_items]
                                        logger.debug(f"Found {len(community_folders)} community folders for {device_name}")

                                    device_data['community_folders'] = community_folders

                                    # Store the device data
                                    manufacturer_devices[device_name] = device_data
                                    manufacturer_device_structure.append(device_name)

                                    logger.debug(f"Loaded device: {device_name} from manufacturer {manufacturer}")
                                else:
                                    logger.warning(f"Device file '{filename}' does not have a device_info.name field")
                            except Exception as e:
                                logger.error(f"Error processing JSON file {filename}: {str(e)}")

            return manufacturer_devices, manufacturer_device_structure

        except Exception as e:
            logger.error(f"Error processing manufacturer {manufacturer}: {str(e)}")
            return {}, []

    def scan_devices(self) -> Dict[str, Dict]:
        """
        Scan devices folder for JSON files and load them into memory
        Returns a dictionary of device names to device data
        """
        logger.info(f"Scanning devices folder: {self.devices_folder}")

        # Clear existing data
        self.devices = {}
        self.manufacturers = []
        self.device_structure = {}

        # Check if devices folder exists
        if not os.path.exists(self.devices_folder):
            logger.warning(f"Devices folder '{self.devices_folder}' does not exist")
            return {}

        # Use the optimized scan implementation
        start_time = time.time()

        try:
            # Get list of manufacturer directories
            manufacturers = [d for d in os.listdir(self.devices_folder) 
                            if os.path.isdir(os.path.join(self.devices_folder, d))]
            logger.info(f"Found {len(manufacturers)} manufacturer directories")
            self.manufacturers = manufacturers

            # Use a thread pool to process manufacturers in parallel
            with concurrent.futures.ThreadPoolExecutor(max_workers=min(32, os.cpu_count() * 4)) as executor:
                # Submit tasks for each manufacturer
                future_to_manufacturer = {
                    executor.submit(self._process_manufacturer, manufacturer): manufacturer
                    for manufacturer in manufacturers
                }

                # Process results as they complete
                for future in concurrent.futures.as_completed(future_to_manufacturer):
                    manufacturer = future_to_manufacturer[future]
                    try:
                        manufacturer_devices, manufacturer_device_structure = future.result()
                        self.devices.update(manufacturer_devices)
                        self.device_structure[manufacturer] = manufacturer_device_structure
                        logger.info(f"Processed manufacturer {manufacturer} with {len(manufacturer_device_structure)} devices")
                    except Exception as e:
                        logger.error(f"Error processing manufacturer {manufacturer}: {str(e)}")

            scan_time = time.time() - start_time
            logger.info(f"Optimized scan completed in {scan_time:.4f} seconds, found {len(self.devices)} devices")

            return self.devices
        except Exception as e:
            scan_time = time.time() - start_time
            logger.error(f"Error during optimized scan: {str(e)} (failed in {scan_time:.4f} seconds)")
            logger.warning("Falling back to standard implementation")

        try:
            # Get list of manufacturer directories
            manufacturers = [d for d in os.listdir(self.devices_folder) 
                            if os.path.isdir(os.path.join(self.devices_folder, d))]
            logger.info(f"Found {len(manufacturers)} manufacturer directories")
            self.manufacturers = manufacturers

            # Process each manufacturer directory
            for manufacturer in manufacturers:
                manufacturer_path = os.path.join(self.devices_folder, manufacturer)
                logger.debug(f"Processing manufacturer directory: {manufacturer}")

                # Initialize device structure for this manufacturer
                self.device_structure[manufacturer] = []

                # Get list of files and directories in manufacturer directory
                items = os.listdir(manufacturer_path)

                # Process device directories and JSON files
                device_dirs = [d for d in items if os.path.isdir(os.path.join(manufacturer_path, d)) and d != 'community']
                json_files = [f for f in items if f.endswith('.json')]

                logger.info(f"Found {len(device_dirs)} device directories and {len(json_files)} JSON files in {manufacturer} directory")

                # Process JSON files directly in manufacturer directory
                for filename in json_files:
                    file_path = os.path.join(manufacturer_path, filename)
                    logger.debug(f"Processing device file: {filename}")

                    try:
                        with open(file_path, 'r') as f:
                            device_data = json.load(f)

                            # Check if device has device_info with a name
                            if 'device_info' in device_data and 'name' in device_data['device_info']:
                                device_name = device_data['device_info']['name']

                                # Add manufacturer information
                                device_data['manufacturer'] = manufacturer

                                # Check for community folders
                                community_path = os.path.join(manufacturer_path, 'community')
                                community_folders = []

                                if os.path.exists(community_path) and os.path.isdir(community_path):
                                    community_items = [f for f in os.listdir(community_path) 
                                                     if f.endswith('.json')]
                                    community_folders = [os.path.splitext(f)[0] for f in community_items]
                                    logger.debug(f"Found {len(community_folders)} community folders for {device_name}")

                                device_data['community_folders'] = community_folders

                                # Store the device data
                                self.devices[device_name] = device_data
                                self.device_structure[manufacturer].append(device_name)

                                logger.info(f"Loaded device: {device_name} from manufacturer {manufacturer}")
                            else:
                                logger.warning(f"Device file '{filename}' does not have a device_info.name field")
                    except json.JSONDecodeError as e:
                        logger.error(f"Invalid JSON in device file '{filename}': {str(e)}")
                    except Exception as e:
                        logger.error(f"Error loading device file '{filename}': {str(e)}")

                # Process device directories
                for device_dir in device_dirs:
                    device_dir_path = os.path.join(manufacturer_path, device_dir)
                    logger.debug(f"Processing device directory: {device_dir}")

                    # Look for JSON files in the device directory
                    device_json_files = [f for f in os.listdir(device_dir_path) if f.endswith('.json')]
                    logger.info(f"Found {len(device_json_files)} JSON files in {device_dir} directory")

                    for filename in device_json_files:
                        file_path = os.path.join(device_dir_path, filename)
                        logger.debug(f"Processing device file: {filename} in directory {device_dir}")

                        try:
                            with open(file_path, 'r') as f:
                                device_data = json.load(f)

                                # Check if device has device_info with a name
                                if 'device_info' in device_data and 'name' in device_data['device_info']:
                                    device_name = device_data['device_info']['name']

                                    # Add manufacturer information
                                    device_data['manufacturer'] = manufacturer

                                    # Check for community folders
                                    community_path = os.path.join(device_dir_path, 'community')
                                    community_folders = []

                                    if os.path.exists(community_path) and os.path.isdir(community_path):
                                        community_items = [f for f in os.listdir(community_path) 
                                                         if f.endswith('.json')]
                                        community_folders = [os.path.splitext(f)[0] for f in community_items]
                                        logger.debug(f"Found {len(community_folders)} community folders for {device_name}")

                                    device_data['community_folders'] = community_folders

                                    # Store the device data
                                    self.devices[device_name] = device_data
                                    self.device_structure[manufacturer].append(device_name)

                                    logger.info(f"Loaded device: {device_name} from manufacturer {manufacturer}")
                                else:
                                    logger.warning(f"Device file '{filename}' in directory {device_dir} does not have a device_info.name field")
                        except json.JSONDecodeError as e:
                            logger.error(f"Invalid JSON in device file '{filename}' in directory {device_dir}: {str(e)}")
                        except Exception as e:
                            logger.error(f"Error loading device file '{filename}' in directory {device_dir}: {str(e)}")

            logger.info(f"Loaded {len(self.devices)} devices from {len(self.manufacturers)} manufacturers")
            return self.devices

        except Exception as e:
            logger.error(f"Error scanning devices folder: {str(e)}")
            return self.devices

    def get_device_by_name(self, name: str) -> Optional[Dict]:
        """Get device data by name"""
        logger.debug(f"Getting device by name: {name}")
        device = self.devices.get(name)
        if device:
            logger.debug(f"Found device: {name}")
        else:
            logger.debug(f"Device not found: {name}")
        return device

    def get_manufacturers(self) -> List[str]:
        """
        Get list of all manufacturers
        Returns a list of manufacturer names
        """
        logger.info(f"Getting all manufacturers ({len(self.manufacturers)} available)")
        return self.manufacturers

    def get_devices_by_manufacturer(self, manufacturer: str) -> List[str]:
        """
        Get list of devices for a specific manufacturer
        Returns a list of device names
        """
        logger.info(f"Getting devices for manufacturer: {manufacturer}")
        return self.device_structure.get(manufacturer, [])

    def get_device_info_by_manufacturer(self, manufacturer: str) -> List[Dict]:
        """
        Get device info for a specific manufacturer
        Returns a list of dictionaries containing device_info
        """
        logger.info(f"Getting device info for manufacturer: {manufacturer}")
        result = []

        # Check if manufacturer exists
        if manufacturer not in self.manufacturers:
            logger.warning(f"Manufacturer not found: {manufacturer}")
            return result

        # Get devices for this manufacturer
        device_names = self.device_structure.get(manufacturer, [])

        # Get device info for each device
        for device_name in device_names:
            device_data = self.devices.get(device_name)
            if device_data and 'device_info' in device_data:
                result.append(device_data['device_info'])

        logger.info(f"Found {len(result)} devices for manufacturer {manufacturer}")
        return result

    def get_community_folders(self, device_name: str) -> List[str]:
        """
        Get list of community folders for a specific device
        Returns a list of folder names
        """
        logger.info(f"Getting community folders for device: {device_name}")
        device = self.devices.get(device_name)
        if device:
            return device.get('community_folders', [])
        return []

    def get_all_devices(self) -> List[Device]:
        """
        Get all devices with their MIDI ports and channels
        Returns a list of Device objects
        """
        logger.info(f"Getting all devices ({len(self.devices)} available)")
        result = []

        try:
            for name, data in self.devices.items():
                try:
                    device_info = data.get('device_info', {})
                    midi_ports = device_info.get('midi_ports', {})
                    midi_channels = device_info.get('midi_channels', {})

                    device = Device(
                        name=name,
                        manufacturer=data.get('manufacturer', ''),
                        midi_port=midi_ports,
                        midi_channel=midi_channels,
                        community_folders=data.get('community_folders', [])
                    )
                    result.append(device)
                    logger.debug(f"Added device: {name}, manufacturer: {data.get('manufacturer', '')}, "
                                f"MIDI ports: {midi_ports}, MIDI channels: {midi_channels}, "
                                f"community folders: {data.get('community_folders', [])}")
                except Exception as e:
                    logger.error(f"Error creating Device object for {name}: {str(e)}")

            logger.info(f"Returning {len(result)} devices")
            return result
        except Exception as e:
            logger.error(f"Error getting all devices: {str(e)}")
            return result

    def clear_cache(self):
        """Clear the JSON cache to force reloading of all files"""
        self._json_cache = {}
        logger.info("JSON cache cleared")

    def _optimized_get_all_patches(self, device_name: Optional[str] = None, community_folder: Optional[str] = None, manufacturer: Optional[str] = None) -> List[Patch]:
        """
        Get all patches from all devices or a specific device with optimized JSON loading

        Args:
            device_name: Optional name of the device to get patches from
            community_folder: Optional name of the community folder to get patches from
            manufacturer: Optional name of the manufacturer to filter devices by

        Returns:
            A list of Patch objects
        """
        if manufacturer and device_name:
            logger.info(f"Getting patches for manufacturer: {manufacturer}, device: {device_name}")
        elif manufacturer:
            logger.info(f"Getting patches for manufacturer: {manufacturer}")
        elif device_name:
            logger.info(f"Getting patches for device: {device_name}")
        else:
            logger.info("Getting all patches from all devices")

        result = []

        try:
            patch_count = 0

            # Filter devices by manufacturer and/or device_name
            devices_to_process = {}

            # If both manufacturer and device_name are provided
            if manufacturer and device_name:
                # Find the device with the matching name and manufacturer
                for name, data in self.devices.items():
                    if name == device_name and data.get('manufacturer') == manufacturer:
                        devices_to_process = {name: data}
                        break

                if not devices_to_process:
                    logger.warning(f"Device not found: {device_name} for manufacturer: {manufacturer}")
                    return []

            # If only manufacturer is provided
            elif manufacturer:
                # Filter devices by manufacturer
                for name, data in self.devices.items():
                    if data.get('manufacturer') == manufacturer:
                        devices_to_process[name] = data

                if not devices_to_process:
                    logger.warning(f"No devices found for manufacturer: {manufacturer}")
                    return []

            # If only device_name is provided
            elif device_name:
                if device_name in self.devices:
                    devices_to_process = {device_name: self.devices[device_name]}
                else:
                    logger.warning(f"Device not found: {device_name}")
                    return []
            else:
                devices_to_process = self.devices

            for device_name, device_data in devices_to_process.items():
                logger.debug(f"Processing device: {device_name}")
                manufacturer = device_data.get('manufacturer', '')

                # Process default presets
                preset_collections = device_data.get('preset_collections', {})
                for collection_name, collection_data in preset_collections.items():
                    presets = collection_data.get('presets', [])
                    logger.debug(f"Device {device_name} collection {collection_name} has {len(presets)} presets")

                    for preset in presets:
                        try:
                            patch = Patch(
                                preset_name=preset.get('preset_name', ''),
                                category=preset.get('category', ''),
                                characters=preset.get('characters', []),
                                sendmidi_command=preset.get('sendmidi_command', ''),
                                cc_0=preset.get('cc_0'),
                                pgm=preset.get('pgm'),
                                source='default'
                            )
                            result.append(patch)
                            patch_count += 1
                            logger.debug(f"Added patch: {patch.preset_name} ({patch.category})")
                        except Exception as e:
                            preset_name = preset.get('preset_name', 'unknown')
                            logger.error(f"Error creating Patch object for {preset_name}: {str(e)}")

                # Process community presets if requested
                if community_folder:
                    logger.debug(f"Processing community folder: {community_folder} for device: {device_name}")

                    # Construct path to community folder
                    community_path = os.path.join(
                        self.devices_folder,
                        manufacturer,
                        'community',
                        f"{community_folder}.json"
                    )

                    if os.path.exists(community_path):
                        try:
                            # Use cached JSON loading
                            community_data = self._load_json_file(community_path)

                            # Process presets from community folder
                            community_presets = community_data.get('presets', [])
                            logger.debug(f"Community folder {community_folder} has {len(community_presets)} presets")

                            for preset in community_presets:
                                try:
                                    patch = Patch(
                                        preset_name=preset.get('preset_name', ''),
                                        category=preset.get('category', ''),
                                        characters=preset.get('characters', []),
                                        sendmidi_command=preset.get('sendmidi_command', ''),
                                        cc_0=preset.get('cc_0'),
                                        pgm=preset.get('pgm'),
                                        source=community_folder
                                    )
                                    result.append(patch)
                                    patch_count += 1
                                    logger.debug(f"Added community patch: {patch.preset_name} ({patch.category})")
                                except Exception as e:
                                    preset_name = preset.get('preset_name', 'unknown')
                                    logger.error(
                                        f"Error creating Patch object for community preset {preset_name}: {str(e)}")
                        except json.JSONDecodeError as e:
                            logger.error(f"Invalid JSON in community file '{community_path}': {str(e)}")
                        except Exception as e:
                            logger.error(f"Error loading community file '{community_path}': {str(e)}")
                    else:
                        logger.warning(f"Community folder not found: {community_path}")

            logger.info(f"Returning {patch_count} patches")
            return result
        except Exception as e:
            logger.error(f"Error getting patches: {str(e)}")
            return result

    def get_all_patches(self, device_name: Optional[str] = None, community_folder: Optional[str] = None, manufacturer: Optional[str] = None) -> List[Patch]:
        """
        Get all patches from all devices or a specific device

        Args:
            device_name: Optional name of the device to get patches from
            community_folder: Optional name of the community folder to get patches from
            manufacturer: Optional name of the manufacturer to filter devices by

        Returns:
            A list of Patch objects
        """
        # Use the optimized version of get_all_patches
        start_time = time.time()
        try:
            # Call the integrated optimized version with the necessary parameters
            result = self._optimized_get_all_patches(
                device_name=device_name,
                community_folder=community_folder,
                manufacturer=manufacturer
            )

            # Log performance metrics
            load_time = time.time() - start_time
            logger.info(f"Returning {len(result)} patches (loaded in {load_time:.4f} seconds using optimized version)")
            return result
        except Exception as e:
            load_time = time.time() - start_time
            logger.error(f"Error getting patches: {str(e)} (failed in {load_time:.4f} seconds)")
            return []

    def get_patch_by_name(self, preset_name: str) -> Optional[Dict]:
        """Get patch data by preset name"""
        start_time = time.time()

        for device_name, device_data in self.devices.items():
            # Check in preset collections
            preset_collections = device_data.get('preset_collections', {})
            for collection_name, collection_data in preset_collections.items():
                presets = collection_data.get('presets', [])
                for preset in presets:
                    if preset.get('preset_name') == preset_name:
                        load_time = time.time() - start_time
                        logger.debug(f"Found preset {preset_name} in device {device_name} collection {collection_name} in {load_time:.4f} seconds")
                        return preset

            # Check in community folders
            manufacturer = device_data.get('manufacturer', '')
            community_folders = device_data.get('community_folders', [])

            for folder in community_folders:
                community_path = os.path.join(
                    self.devices_folder, 
                    manufacturer, 
                    'community', 
                    f"{folder}.json"
                )

                if os.path.exists(community_path):
                    try:
                        # Use cached JSON loading for better performance
                        community_data = self._load_json_file(community_path)
                        community_presets = community_data.get('presets', [])

                        for preset in community_presets:
                            if preset.get('preset_name') == preset_name:
                                load_time = time.time() - start_time
                                logger.debug(f"Found preset {preset_name} in community folder {folder} in {load_time:.4f} seconds")
                                return preset
                    except Exception as e:
                        logger.error(f"Error loading community file '{community_path}': {str(e)}")

        load_time = time.time() - start_time
        logger.debug(f"Preset {preset_name} not found (searched in {load_time:.4f} seconds)")
        return None

    def check_directory_structure(self, manufacturer: str, device: str, create_if_missing: bool = True) -> DirectoryStructureResponse:
        """
        Check if the manufacturer and device directories exist, and if the device JSON file exists.
        Optionally create them if they don't exist.

        Args:
            manufacturer: Name of the manufacturer
            device: Name of the device
            create_if_missing: Whether to create the directories and file if they don't exist

        Returns:
            DirectoryStructureResponse object with information about the directory structure
        """
        # Normalize names to avoid path issues
        manufacturer_safe = manufacturer.replace(' ', '_')
        device_safe = device.replace(' ', '_')

        # Construct paths
        manufacturer_path = os.path.join(self.devices_folder, manufacturer_safe)
        device_path = os.path.join(manufacturer_path, device_safe)
        json_path = os.path.join(device_path, f"{manufacturer_safe.lower()}_{device_safe.lower()}.json")

        # Check if they exist
        manufacturer_exists = os.path.exists(manufacturer_path) and os.path.isdir(manufacturer_path)
        device_exists = os.path.exists(device_path) and os.path.isdir(device_path)
        json_exists = os.path.exists(json_path)

        response = DirectoryStructureResponse(
            manufacturer_exists=manufacturer_exists,
            device_exists=device_exists,
            json_exists=json_exists,
            json_path=json_path
        )

        # Create if requested and missing
        if create_if_missing:
            # Create manufacturer directory if it doesn't exist
            if not manufacturer_exists:
                try:
                    os.makedirs(manufacturer_path, exist_ok=True)
                    response.created_manufacturer = True
                    manufacturer_exists = True
                    response.manufacturer_exists = True
                    logger.info(f"Created manufacturer directory: {manufacturer_path}")
                except Exception as e:
                    logger.error(f"Error creating manufacturer directory {manufacturer_path}: {str(e)}")
                    return response

            # Create device directory if it doesn't exist
            if manufacturer_exists and not device_exists:
                try:
                    os.makedirs(device_path, exist_ok=True)
                    response.created_device = True
                    device_exists = True
                    response.device_exists = True
                    logger.info(f"Created device directory: {device_path}")
                except Exception as e:
                    logger.error(f"Error creating device directory {device_path}: {str(e)}")
                    return response

            # Create JSON file if it doesn't exist
            if device_exists and not json_exists:
                try:
                    # Create a basic device JSON file
                    device_data = {
                        "_metadata": {
                            "schema_version": "1.0.0",
                            "file_revision": 1,
                            "created_by": "r2midi",
                            "modified_by": "r2midi",
                            "created_at": datetime.now().isoformat(),
                            "modified_at": datetime.now().isoformat(),
                            "migration_path": [],
                            "compatibility": {}
                        },
                        "device_info": {
                            "name": device,
                            "version": "1.0.0",
                            "manufacturer": manufacturer,
                            "manufacturer_id": 0,
                            "device_id": 0,
                            "ports": ["IN", "OUT"],
                            "midi_channels": {"IN": 1, "OUT": 1},
                            "midi_ports": {"IN": "", "OUT": ""}
                        },
                        "capabilities": {},
                        "preset_collections": {
                            "default": {
                                "metadata": {
                                    "name": "default",
                                    "version": "1.0",
                                    "revision": 1,
                                    "author": "r2midi",
                                    "description": "Default preset collection",
                                    "readonly": False,
                                    "preset_count": 0,
                                    "parent_collections": [],
                                    "sync_status": "synced",
                                    "created_at": datetime.now().isoformat(),
                                    "modified_at": datetime.now().isoformat()
                                },
                                "presets": [],
                                "preset_metadata": {}
                            }
                        }
                    }

                    with open(json_path, 'w') as f:
                        json.dump(device_data, f, indent=2)

                    response.created_json = True
                    response.json_exists = True
                    logger.info(f"Created device JSON file: {json_path}")
                except Exception as e:
                    logger.error(f"Error creating device JSON file {json_path}: {str(e)}")
                    return response

        return response

    def create_manufacturer(self, name: str) -> Tuple[bool, str]:
        """
        Create a new manufacturer directory

        Args:
            name: Name of the manufacturer

        Returns:
            Tuple of (success, message)
        """
        # Normalize name to avoid path issues
        name_safe = name.replace(' ', '_')

        # Construct path
        manufacturer_path = os.path.join(self.devices_folder, name_safe)

        # Check if it already exists
        if os.path.exists(manufacturer_path):
            return False, f"Manufacturer '{name}' already exists"

        # Create the directory
        try:
            os.makedirs(manufacturer_path, exist_ok=True)
            logger.info(f"Created manufacturer directory: {manufacturer_path}")

            # Rescan devices to include the new manufacturer
            self.scan_devices()

            return True, f"Manufacturer '{name}' created successfully"
        except Exception as e:
            logger.error(f"Error creating manufacturer directory {manufacturer_path}: {str(e)}")
            return False, f"Error creating manufacturer: {str(e)}"

    def delete_manufacturer(self, name: str) -> Tuple[bool, str]:
        """
        Delete a manufacturer directory and all its contents

        Args:
            name: Name of the manufacturer

        Returns:
            Tuple of (success, message)
        """
        # Check if manufacturer exists
        if name not in self.manufacturers:
            return False, f"Manufacturer '{name}' does not exist"

        # Construct path
        manufacturer_path = os.path.join(self.devices_folder, name)

        # Check if it exists
        if not os.path.exists(manufacturer_path):
            return False, f"Manufacturer directory '{manufacturer_path}' does not exist"

        # Delete the directory
        try:
            shutil.rmtree(manufacturer_path)
            logger.info(f"Deleted manufacturer directory: {manufacturer_path}")

            # Rescan devices to update the list
            self.scan_devices()

            return True, f"Manufacturer '{name}' deleted successfully"
        except Exception as e:
            logger.error(f"Error deleting manufacturer directory {manufacturer_path}: {str(e)}")
            return False, f"Error deleting manufacturer: {str(e)}"

    def create_device(self, device_data: Dict[str, Any]) -> Tuple[bool, str, Optional[str]]:
        """
        Create a new device with the given data

        Args:
            device_data: Dictionary containing device data

        Returns:
            Tuple of (success, message, json_path)
        """
        manufacturer = device_data.get('manufacturer')
        name = device_data.get('name')

        if not manufacturer or not name:
            return False, "Manufacturer and name are required", None

        # Check if manufacturer exists
        if manufacturer not in self.manufacturers:
            # Create manufacturer if it doesn't exist
            success, message = self.create_manufacturer(manufacturer)
            if not success:
                return False, f"Error creating manufacturer: {message}", None

        # Check and create directory structure
        response = self.check_directory_structure(manufacturer, name, create_if_missing=True)

        if not response.manufacturer_exists or not response.device_exists:
            return False, "Failed to create directory structure", None

        # Create or update the device JSON file
        try:
            # Normalize names for file paths
            manufacturer_safe = manufacturer.replace(' ', '_')
            device_safe = name.replace(' ', '_')

            # Construct the JSON file path
            json_path = os.path.join(
                self.devices_folder,
                manufacturer_safe,
                device_safe,
                f"{manufacturer_safe.lower()}_{device_safe.lower()}.json"
            )

            # Create a basic device JSON file
            device_json = {
                "_metadata": {
                    "schema_version": "1.0.0",
                    "file_revision": 1,
                    "created_by": "r2midi",
                    "modified_by": "r2midi",
                    "created_at": datetime.now().isoformat(),
                    "modified_at": datetime.now().isoformat(),
                    "migration_path": [],
                    "compatibility": {}
                },
                "device_info": {
                    "name": name,
                    "version": device_data.get('version', "1.0.0"),
                    "manufacturer": manufacturer,
                    "manufacturer_id": device_data.get('manufacturer_id', 0),
                    "device_id": device_data.get('device_id', 0),
                    "ports": ["IN", "OUT"],
                    "midi_channels": device_data.get('midi_channels', {"IN": 1, "OUT": 1}),
                    "midi_ports": device_data.get('midi_ports', {"IN": "", "OUT": ""})
                },
                "capabilities": {},
                "preset_collections": {
                    "default": {
                        "metadata": {
                            "name": "default",
                            "version": "1.0",
                            "revision": 1,
                            "author": "r2midi",
                            "description": "Default preset collection",
                            "readonly": False,
                            "preset_count": 0,
                            "parent_collections": [],
                            "sync_status": "synced",
                            "created_at": datetime.now().isoformat(),
                            "modified_at": datetime.now().isoformat()
                        },
                        "presets": [],
                        "preset_metadata": {}
                    }
                }
            }

            with open(json_path, 'w') as f:
                json.dump(device_json, f, indent=2)

            logger.info(f"Created device JSON file: {json_path}")

            # Rescan devices to include the new device
            self.scan_devices()

            return True, f"Device '{name}' created successfully", json_path
        except Exception as e:
            logger.error(f"Error creating device JSON file: {str(e)}")
            return False, f"Error creating device: {str(e)}", None

    def delete_device(self, manufacturer: str, device: str) -> Tuple[bool, str]:
        """
        Delete a device directory and all its contents

        Args:
            manufacturer: Name of the manufacturer
            device: Name of the device

        Returns:
            Tuple of (success, message)
        """
        # Check if manufacturer exists
        if manufacturer not in self.manufacturers:
            return False, f"Manufacturer '{manufacturer}' does not exist"

        # Check if device exists
        devices = self.get_devices_by_manufacturer(manufacturer)
        if device not in devices:
            return False, f"Device '{device}' does not exist for manufacturer '{manufacturer}'"

        # Construct path
        device_path = os.path.join(self.devices_folder, manufacturer, device)

        # Check if it exists
        if not os.path.exists(device_path):
            return False, f"Device directory '{device_path}' does not exist"

        # Delete the directory
        try:
            shutil.rmtree(device_path)
            logger.info(f"Deleted device directory: {device_path}")

            # Rescan devices to update the list
            self.scan_devices()

            return True, f"Device '{device}' deleted successfully"
        except Exception as e:
            logger.error(f"Error deleting device directory {device_path}: {str(e)}")
            return False, f"Error deleting device: {str(e)}"

    def create_preset(self, preset_data: Dict[str, Any]) -> Tuple[bool, str]:
        """
        Create a new preset in the specified collection

        Args:
            preset_data: Dictionary containing preset data

        Returns:
            Tuple of (success, message)
        """
        manufacturer = preset_data.get('manufacturer')
        device_name = preset_data.get('device')
        collection_name = preset_data.get('collection')
        preset_name = preset_data.get('preset_name')

        if not manufacturer or not device_name or not collection_name or not preset_name:
            return False, "Manufacturer, device, collection, and preset_name are required"

        # Check if manufacturer exists
        if manufacturer not in self.manufacturers:
            return False, f"Manufacturer '{manufacturer}' does not exist"

        # Check if device exists
        devices = self.get_devices_by_manufacturer(manufacturer)
        if device_name not in devices:
            return False, f"Device '{device_name}' does not exist for manufacturer '{manufacturer}'"

        # Get the device data
        device = self.get_device_by_name(device_name)
        if not device:
            return False, f"Device '{device_name}' not found"

        # Check if preset_collections exists, create it if it doesn't
        if 'preset_collections' not in device:
            device['preset_collections'] = {}
            logger.info(f"Created preset_collections for device '{device_name}'")

        # Check if collection exists, create it if it doesn't
        preset_collections = device.get('preset_collections', {})
        if collection_name not in preset_collections:
            # Create the collection
            collection_display_name = "Factory Presets" if collection_name == "factory_presets" else collection_name
            preset_collections[collection_name] = {
                "metadata": {
                    "name": collection_display_name,
                    "version": "1.0",
                    "revision": 1,
                    "author": "r2midi",
                    "description": f"{collection_display_name} for {device_name}",
                    "readonly": False,
                    "preset_count": 0,
                    "parent_collections": [],
                    "sync_status": "synced",
                    "created_at": datetime.now().isoformat(),
                    "modified_at": datetime.now().isoformat()
                },
                "presets": [],
                "preset_metadata": {}
            }
            logger.info(f"Created collection '{collection_name}' for device '{device_name}'")

            # Update the device data
            device['preset_collections'] = preset_collections

        # Get the collection data
        collection = preset_collections[collection_name]
        presets = collection.get('presets', [])
        preset_metadata = collection.get('preset_metadata', {})

        # Check if preset already exists
        for preset in presets:
            if preset.get('preset_name') == preset_name:
                return False, f"Preset '{preset_name}' already exists in collection '{collection_name}'"

        # Create the preset
        try:
            # Generate a unique preset ID
            preset_id = f"{preset_name.lower().replace(' ', '_')}_{len(presets) + 1}"

            # Create the preset
            new_preset = {
                "preset_id": preset_id,
                "preset_name": preset_name,
                "category": preset_data.get('category', ""),
                "cc_0": preset_data.get('cc_0'),
                "pgm": preset_data.get('pgm', 0),
                "characters": preset_data.get('characters', []),
                "sendmidi_command": preset_data.get('sendmidi_command', f"sendmidi dev \"{device_name}\" cc 0 {preset_data.get('cc_0', 0)} pc {preset_data.get('pgm', 0)}")
            }

            # Create the preset metadata
            new_preset_metadata = {
                "version": "1.0",
                "validation_status": "validated",
                "source": "user",
                "created_at": datetime.now().isoformat(),
                "modified_at": datetime.now().isoformat()
            }

            # Add the preset to the collection
            presets.append(new_preset)
            preset_metadata[preset_id] = new_preset_metadata

            # Update the collection metadata
            collection['metadata']['preset_count'] = len(presets)
            collection['metadata']['modified_at'] = datetime.now().isoformat()

            # Update the device data
            device['preset_collections'][collection_name]['presets'] = presets
            device['preset_collections'][collection_name]['preset_metadata'] = preset_metadata
            device['preset_collections'][collection_name]['metadata'] = collection['metadata']

            # Save the device data
            device_path = os.path.join(self.devices_folder, manufacturer, device_name)
            json_files = [f for f in os.listdir(device_path) if f.endswith('.json')]

            if not json_files:
                return False, f"No JSON file found for device '{device_name}'"

            json_path = os.path.join(device_path, json_files[0])

            with open(json_path, 'w') as f:
                json.dump(device, f, indent=2)

            logger.info(f"Created preset '{preset_name}' in collection '{collection_name}' for device '{device_name}'")

            return True, f"Preset '{preset_name}' created successfully"
        except Exception as e:
            logger.error(f"Error creating preset: {str(e)}")
            return False, f"Error creating preset: {str(e)}"

    def update_preset(self, preset_data: Dict[str, Any]) -> Tuple[bool, str]:
        """
        Update an existing preset with new data

        Args:
            preset_data: Dictionary containing preset data

        Returns:
            Tuple of (success, message)
        """
        manufacturer = preset_data.get('manufacturer')
        device_name = preset_data.get('device')
        collection_name = preset_data.get('collection')
        preset_name = preset_data.get('preset_name')

        if not manufacturer or not device_name or not collection_name or not preset_name:
            return False, "Manufacturer, device, collection, and preset_name are required"

        # Check if manufacturer exists
        if manufacturer not in self.manufacturers:
            return False, f"Manufacturer '{manufacturer}' does not exist"

        # Check if device exists
        devices = self.get_devices_by_manufacturer(manufacturer)
        if device_name not in devices:
            return False, f"Device '{device_name}' does not exist for manufacturer '{manufacturer}'"

        # Get the device data
        device = self.get_device_by_name(device_name)
        if not device:
            return False, f"Device '{device_name}' not found"

        # Check if collection exists
        preset_collections = device.get('preset_collections', {})
        if collection_name not in preset_collections:
            return False, f"Collection '{collection_name}' does not exist for device '{device_name}'"

        # Get the collection data
        collection = preset_collections[collection_name]
        presets = collection.get('presets', [])
        preset_metadata = collection.get('preset_metadata', {})

        # Find the preset
        preset_index = None
        preset_id = None

        for i, preset in enumerate(presets):
            if preset.get('preset_name') == preset_name:
                preset_index = i
                preset_id = preset.get('preset_id')
                break

        if preset_index is None:
            return False, f"Preset '{preset_name}' not found in collection '{collection_name}'"

        # Update the preset
        try:
            # Create the updated preset
            updated_preset = {
                "preset_id": preset_id,
                "preset_name": preset_name,
                "category": preset_data.get('category', ""),
                "cc_0": preset_data.get('cc_0'),
                "pgm": preset_data.get('pgm', 0),
                "characters": preset_data.get('characters', []),
                "sendmidi_command": preset_data.get('sendmidi_command', f"sendmidi dev \"{device_name}\" cc 0 {preset_data.get('cc_0', 0)} pc {preset_data.get('pgm', 0)}")
            }

            # Update the preset metadata
            if preset_id and preset_id in preset_metadata:
                preset_metadata[preset_id]["modified_at"] = datetime.now().isoformat()

            # Update the preset in the collection
            presets[preset_index] = updated_preset

            # Update the collection metadata
            collection['metadata']['modified_at'] = datetime.now().isoformat()

            # Update the device data
            device['preset_collections'][collection_name]['presets'] = presets
            device['preset_collections'][collection_name]['preset_metadata'] = preset_metadata
            device['preset_collections'][collection_name]['metadata'] = collection['metadata']

            # Save the device data
            device_path = os.path.join(self.devices_folder, manufacturer, device_name)
            json_files = [f for f in os.listdir(device_path) if f.endswith('.json')]

            if not json_files:
                return False, f"No JSON file found for device '{device_name}'"

            json_path = os.path.join(device_path, json_files[0])

            with open(json_path, 'w') as f:
                json.dump(device, f, indent=2)

            logger.info(f"Updated preset '{preset_name}' in collection '{collection_name}' for device '{device_name}'")

            return True, f"Preset '{preset_name}' updated successfully"
        except Exception as e:
            logger.error(f"Error updating preset: {str(e)}")
            return False, f"Error updating preset: {str(e)}"

    def delete_preset(self, manufacturer: str, device_name: str, collection_name: str, preset_name: str) -> Tuple[bool, str]:
        """
        Delete a preset from the specified collection

        Args:
            manufacturer: Name of the manufacturer
            device_name: Name of the device
            collection_name: Name of the collection
            preset_name: Name of the preset

        Returns:
            Tuple of (success, message)
        """
        # Check if manufacturer exists
        if manufacturer not in self.manufacturers:
            return False, f"Manufacturer '{manufacturer}' does not exist"

        # Check if device exists
        devices = self.get_devices_by_manufacturer(manufacturer)
        if device_name not in devices:
            return False, f"Device '{device_name}' does not exist for manufacturer '{manufacturer}'"

        # Get the device data
        device = self.get_device_by_name(device_name)
        if not device:
            return False, f"Device '{device_name}' not found"

        # Check if collection exists
        preset_collections = device.get('preset_collections', {})
        if collection_name not in preset_collections:
            return False, f"Collection '{collection_name}' does not exist for device '{device_name}'"

        # Get the collection data
        collection = preset_collections[collection_name]
        presets = collection.get('presets', [])
        preset_metadata = collection.get('preset_metadata', {})

        # Find the preset
        preset_index = None
        preset_id = None

        for i, preset in enumerate(presets):
            if preset.get('preset_name') == preset_name:
                preset_index = i
                preset_id = preset.get('preset_id')
                break

        if preset_index is None:
            return False, f"Preset '{preset_name}' not found in collection '{collection_name}'"

        # Delete the preset
        try:
            # Remove the preset from the collection
            del presets[preset_index]

            # Remove the preset metadata if it exists
            if preset_id and preset_id in preset_metadata:
                del preset_metadata[preset_id]

            # Update the collection metadata
            collection['metadata']['preset_count'] = len(presets)
            collection['metadata']['modified_at'] = datetime.now().isoformat()

            # Update the device data
            device['preset_collections'][collection_name]['presets'] = presets
            device['preset_collections'][collection_name]['preset_metadata'] = preset_metadata
            device['preset_collections'][collection_name]['metadata'] = collection['metadata']

            # Save the device data
            device_path = os.path.join(self.devices_folder, manufacturer, device_name)
            json_files = [f for f in os.listdir(device_path) if f.endswith('.json')]

            if not json_files:
                return False, f"No JSON file found for device '{device_name}'"

            json_path = os.path.join(device_path, json_files[0])

            with open(json_path, 'w') as f:
                json.dump(device, f, indent=2)

            logger.info(f"Deleted preset '{preset_name}' from collection '{collection_name}' for device '{device_name}'")

            return True, f"Preset '{preset_name}' deleted successfully"
        except Exception as e:
            logger.error(f"Error deleting preset: {str(e)}")
            return False, f"Error deleting preset: {str(e)}"
