from kubernetes import client, config
from kubernetes.client.rest import ApiException
from typing import Dict, List, Optional, Any
import yaml
import time
import base64
import boto3
import json
from botocore.exceptions import ClientError, NoCredentialsError


class EKSClient:
    """AWS EKS client for cluster management"""
    
    def __init__(self, region: str = "us-west-2"):
        """Initialize EKS client
        
        Args:
            region: AWS region for EKS operations
        """
        self.region = region
        try:
            self.eks_client = boto3.client('eks', region_name=region)
            self.ec2_client = boto3.client('ec2', region_name=region)
            self.iam_client = boto3.client('iam', region_name=region)
        except (NoCredentialsError, ClientError) as e:
            raise Exception(f"AWS credentials not found or invalid: {e}")
    
    def create_cluster(self, cluster_name: str, version: str = "1.29", 
                      subnets: List[str] = None, security_groups: List[str] = None,
                      role_arn: str = None, node_group_name: str = None,
                      instance_types: List[str] = None, ami_type: str = "AL2_x86_64",
                      capacity_type: str = "ON_DEMAND", scaling_config: Dict = None,
                      create_nodegroup: bool = True, wait_for_cluster: bool = False) -> Dict:
        """Create an EKS cluster
        
        Args:
            cluster_name: Name of the EKS cluster
            version: Kubernetes version
            subnets: List of subnet IDs
            security_groups: List of security group IDs
            role_arn: IAM role ARN for the cluster
            node_group_name: Name for the node group
            instance_types: List of EC2 instance types
            ami_type: AMI type for nodes
            capacity_type: Capacity type (ON_DEMAND or SPOT)
            scaling_config: Scaling configuration for node group
            create_nodegroup: Whether to create a node group automatically
            wait_for_cluster: Whether to wait for cluster to be active before returning
            
        Returns:
            Dict containing cluster information
        """
        try:
            # Use default values if not provided
            if subnets is None:
                subnets = self._get_default_subnets()
            
            if role_arn is None:
                role_arn = self._create_or_get_cluster_role()
            
            if instance_types is None:
                instance_types = ["t3.medium"]
            
            if scaling_config is None:
                scaling_config = {
                    "minSize": 1,
                    "maxSize": 3,
                    "desiredSize": 2
                }
            
            # Create cluster
            cluster_response = self.eks_client.create_cluster(
                name=cluster_name,
                version=version,
                roleArn=role_arn,
                resourcesVpcConfig={
                    'subnetIds': subnets,
                    'securityGroupIds': security_groups or [],
                    'endpointPublicAccess': True,
                    'endpointPrivateAccess': True
                },
                logging={
                    'clusterLogging': [
                        {
                            'types': ['api', 'audit', 'authenticator', 'controllerManager', 'scheduler'],
                            'enabled': True
                        }
                    ]
                }
            )
            
            cluster_info = {
                'cluster_name': cluster_name,
                'status': 'CREATING',
                'cluster_arn': cluster_response['cluster']['arn'],
                'endpoint': cluster_response['cluster'].get('endpoint', 'Not available yet'),
                'version': version,
                'role_arn': role_arn,
                'subnets': subnets,
                'created_at': cluster_response['cluster']['createdAt']
            }
            
            # Create node group if requested
            if create_nodegroup:
                if node_group_name is None:
                    node_group_name = f"{cluster_name}-nodegroup"
                
                cluster_info['node_group_name'] = node_group_name
                cluster_info['instance_types'] = instance_types
                cluster_info['scaling_config'] = scaling_config
                cluster_info['create_nodegroup'] = True
                
                # If wait_for_cluster is True, create the node group after cluster is active
                if wait_for_cluster:
                    if self.wait_for_cluster_active(cluster_name):
                        try:
                            nodegroup_info = self.create_nodegroup(
                                cluster_name=cluster_name,
                                nodegroup_name=node_group_name,
                                instance_types=instance_types,
                                ami_type=ami_type,
                                capacity_type=capacity_type,
                                scaling_config=scaling_config,
                                subnets=subnets
                            )
                            cluster_info['nodegroup_info'] = nodegroup_info
                        except Exception as e:
                            cluster_info['nodegroup_error'] = str(e)
                    else:
                        cluster_info['nodegroup_error'] = "Cluster creation timed out"
            
            return cluster_info
            
        except ClientError as e:
            raise Exception(f"Failed to create EKS cluster: {e}")
    
    def _get_default_subnets(self) -> List[str]:
        """Get default subnets for EKS cluster from different AZs"""
        try:
            response = self.ec2_client.describe_subnets()
            
            # Group subnets by availability zone
            subnets_by_az = {}
            for subnet in response['Subnets']:
                if subnet['State'] == 'available':
                    az = subnet['AvailabilityZone']
                    if az not in subnets_by_az:
                        subnets_by_az[az] = []
                    subnets_by_az[az].append(subnet['SubnetId'])
            
            # Get at least 2 subnets from different AZs
            selected_subnets = []
            for az, subnet_ids in subnets_by_az.items():
                if len(selected_subnets) < 2:
                    selected_subnets.append(subnet_ids[0])  # Take first subnet from each AZ
            
            if len(selected_subnets) < 2:
                # If we don't have subnets in 2 different AZs, let's create them
                selected_subnets = self._create_default_vpc_subnets()
            
            return selected_subnets
            
        except ClientError as e:
            raise Exception(f"Failed to get default subnets: {e}")
    
    def _create_default_vpc_subnets(self) -> List[str]:
        """Create default VPC and subnets for EKS if none exist"""
        try:
            # Get default VPC
            vpcs = self.ec2_client.describe_vpcs(Filters=[{'Name': 'isDefault', 'Values': ['true']}])
            if not vpcs['Vpcs']:
                raise Exception("No default VPC found. Please create subnets manually or set up a VPC.")
            
            vpc_id = vpcs['Vpcs'][0]['VpcId']
            
            # Get available AZs
            azs = self.ec2_client.describe_availability_zones()
            if len(azs['AvailabilityZones']) < 2:
                raise Exception("Need at least 2 availability zones for EKS cluster")
            
            # Create subnets in first 2 AZs
            subnet_ids = []
            for i, az in enumerate(azs['AvailabilityZones'][:2]):
                cidr = f"172.31.{i * 16}.0/20"  # Create non-overlapping CIDR blocks
                
                try:
                    response = self.ec2_client.create_subnet(
                        VpcId=vpc_id,
                        CidrBlock=cidr,
                        AvailabilityZone=az['ZoneName']
                    )
                    subnet_id = response['Subnet']['SubnetId']
                    subnet_ids.append(subnet_id)
                    
                    # Enable auto-assign public IP
                    self.ec2_client.modify_subnet_attribute(
                        SubnetId=subnet_id,
                        MapPublicIpOnLaunch={'Value': True}
                    )
                    
                    # Tag the subnet
                    self.ec2_client.create_tags(
                        Resources=[subnet_id],
                        Tags=[
                            {'Key': 'Name', 'Value': f'eks-subnet-{az["ZoneName"]}'},
                            {'Key': 'kubernetes.io/role/elb', 'Value': '1'}
                        ]
                    )
                    
                except ClientError as e:
                    if e.response['Error']['Code'] == 'InvalidVpc.Range':
                        # Try a different CIDR range
                        cidr = f"10.0.{i}.0/24"
                        response = self.ec2_client.create_subnet(
                            VpcId=vpc_id,
                            CidrBlock=cidr,
                            AvailabilityZone=az['ZoneName']
                        )
                        subnet_id = response['Subnet']['SubnetId']
                        subnet_ids.append(subnet_id)
                        
                        # Enable auto-assign public IP
                        self.ec2_client.modify_subnet_attribute(
                            SubnetId=subnet_id,
                            MapPublicIpOnLaunch={'Value': True}
                        )
                        
                        # Tag the subnet
                        self.ec2_client.create_tags(
                            Resources=[subnet_id],
                            Tags=[
                                {'Key': 'Name', 'Value': f'eks-subnet-{az["ZoneName"]}'},
                                {'Key': 'kubernetes.io/role/elb', 'Value': '1'}
                            ]
                        )
            
            return subnet_ids
            
        except ClientError as e:
            raise Exception(f"Failed to create default subnets: {e}")
    
    def _create_or_get_cluster_role(self) -> str:
        """Create or get IAM role for EKS cluster"""
        role_name = "eks-cluster-role"
        
        try:
            # Check if role exists
            response = self.iam_client.get_role(RoleName=role_name)
            return response['Role']['Arn']
            
        except ClientError as e:
            if e.response['Error']['Code'] == 'NoSuchEntity':
                # Create the role
                trust_policy = {
                    "Version": "2012-10-17",
                    "Statement": [
                        {
                            "Effect": "Allow",
                            "Principal": {
                                "Service": "eks.amazonaws.com"
                            },
                            "Action": "sts:AssumeRole"
                        }
                    ]
                }
                
                response = self.iam_client.create_role(
                    RoleName=role_name,
                    AssumeRolePolicyDocument=json.dumps(trust_policy),
                    Description="EKS cluster role created by k8s-helper"
                )
                
                # Attach required policies
                policies = [
                    "arn:aws:iam::aws:policy/AmazonEKSClusterPolicy"
                ]
                
                for policy in policies:
                    self.iam_client.attach_role_policy(
                        RoleName=role_name,
                        PolicyArn=policy
                    )
                
                return response['Role']['Arn']
            else:
                raise Exception(f"Failed to create or get cluster role: {e}")
    
    def _create_or_get_nodegroup_role(self) -> str:
        """Create or get IAM role for EKS node group"""
        role_name = "eks-nodegroup-role"
        
        try:
            # Check if role exists
            response = self.iam_client.get_role(RoleName=role_name)
            return response['Role']['Arn']
            
        except ClientError as e:
            if e.response['Error']['Code'] == 'NoSuchEntity':
                # Create the role
                trust_policy = {
                    "Version": "2012-10-17",
                    "Statement": [
                        {
                            "Effect": "Allow",
                            "Principal": {
                                "Service": "ec2.amazonaws.com"
                            },
                            "Action": "sts:AssumeRole"
                        }
                    ]
                }
                
                response = self.iam_client.create_role(
                    RoleName=role_name,
                    AssumeRolePolicyDocument=json.dumps(trust_policy),
                    Description="EKS node group role created by k8s-helper"
                )
                
                # Attach required policies for node group
                policies = [
                    "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy",
                    "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy",
                    "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly"
                ]
                
                for policy in policies:
                    self.iam_client.attach_role_policy(
                        RoleName=role_name,
                        PolicyArn=policy
                    )
                
                return response['Role']['Arn']
            else:
                raise Exception(f"Failed to create or get node group role: {e}")
    
    def create_nodegroup(self, cluster_name: str, nodegroup_name: str, 
                        instance_types: List[str] = None, ami_type: str = "AL2_x86_64",
                        capacity_type: str = "ON_DEMAND", scaling_config: Dict = None,
                        subnets: List[str] = None, node_role_arn: str = None,
                        ssh_key: str = None) -> Dict:
        """Create an EKS managed node group
        
        Args:
            cluster_name: Name of the EKS cluster
            nodegroup_name: Name of the node group
            instance_types: List of EC2 instance types
            ami_type: AMI type for nodes
            capacity_type: Capacity type (ON_DEMAND or SPOT)
            scaling_config: Scaling configuration
            subnets: List of subnet IDs
            node_role_arn: IAM role ARN for nodes
            ssh_key: EC2 SSH key name for remote access (optional)
            
        Returns:
            Dict containing node group information
        """
        try:
            # Use defaults if not provided
            if instance_types is None:
                instance_types = ["t3.medium"]
            
            if scaling_config is None:
                scaling_config = {
                    "minSize": 1,
                    "maxSize": 3,
                    "desiredSize": 2
                }
            
            if node_role_arn is None:
                node_role_arn = self._create_or_get_nodegroup_role()
            
            if subnets is None:
                # Get cluster's subnets
                cluster_info = self.get_cluster_status(cluster_name)
                cluster_details = self.eks_client.describe_cluster(name=cluster_name)
                subnets = cluster_details['cluster']['resourcesVpcConfig']['subnetIds']
            
            # Prepare node group creation parameters
            nodegroup_params = {
                'clusterName': cluster_name,
                'nodegroupName': nodegroup_name,
                'scalingConfig': scaling_config,
                'instanceTypes': instance_types,
                'amiType': ami_type,
                'capacityType': capacity_type,
                'nodeRole': node_role_arn,
                'subnets': subnets
            }
            
            # Only add remoteAccess if SSH key is provided
            if ssh_key:
                nodegroup_params['remoteAccess'] = {
                    'ec2SshKey': ssh_key
                }
            
            # Create the node group
            response = self.eks_client.create_nodegroup(**nodegroup_params)
            
            return {
                'nodegroup_name': nodegroup_name,
                'cluster_name': cluster_name,
                'status': 'CREATING',
                'nodegroup_arn': response['nodegroup']['nodegroupArn'],
                'instance_types': instance_types,
                'ami_type': ami_type,
                'capacity_type': capacity_type,
                'scaling_config': scaling_config,
                'created_at': response['nodegroup']['createdAt'],
                'node_role_arn': node_role_arn
            }
            
        except ClientError as e:
            raise Exception(f"Failed to create node group: {e}")
    
    def get_nodegroup_status(self, cluster_name: str, nodegroup_name: str) -> Dict:
        """Get EKS node group status"""
        try:
            response = self.eks_client.describe_nodegroup(
                clusterName=cluster_name,
                nodegroupName=nodegroup_name
            )
            nodegroup = response['nodegroup']
            
            return {
                'name': nodegroup['nodegroupName'],
                'cluster_name': cluster_name,
                'status': nodegroup['status'],
                'instance_types': nodegroup['instanceTypes'],
                'ami_type': nodegroup['amiType'],
                'capacity_type': nodegroup['capacityType'],
                'scaling_config': nodegroup['scalingConfig'],
                'created_at': nodegroup['createdAt'],
                'arn': nodegroup['nodegroupArn']
            }
            
        except ClientError as e:
            raise Exception(f"Failed to get node group status: {e}")
    
    def wait_for_nodegroup_active(self, cluster_name: str, nodegroup_name: str, timeout: int = 1200) -> bool:
        """Wait for EKS node group to become active"""
        import time
        start_time = time.time()
        
        while time.time() - start_time < timeout:
            try:
                status = self.get_nodegroup_status(cluster_name, nodegroup_name)
                if status['status'] == 'ACTIVE':
                    return True
                elif status['status'] in ['CREATE_FAILED', 'DEGRADED']:
                    raise Exception(f"Node group creation failed with status: {status['status']}")
                
                time.sleep(30)  # Check every 30 seconds
                
            except Exception as e:
                raise Exception(f"Error waiting for node group: {e}")
        
        return False
    
    def list_nodegroups(self, cluster_name: str) -> List[Dict]:
        """List all node groups for a cluster"""
        try:
            response = self.eks_client.list_nodegroups(clusterName=cluster_name)
            nodegroups = []
            
            for nodegroup_name in response['nodegroups']:
                try:
                    nodegroup_info = self.get_nodegroup_status(cluster_name, nodegroup_name)
                    nodegroups.append(nodegroup_info)
                except Exception as e:
                    # Skip node groups that can't be described
                    continue
            
            return nodegroups
            
        except ClientError as e:
            raise Exception(f"Failed to list node groups: {e}")
    
    def create_nodegroup_with_instance_profile(self, cluster_name: str, nodegroup_name: str, 
                                              instance_types: List[str] = None, ami_type: str = "AL2_x86_64",
                                              capacity_type: str = "ON_DEMAND", scaling_config: Dict = None,
                                              subnets: List[str] = None, node_role_arn: str = None,
                                              instance_profile: str = None) -> Dict:
        """Create an EKS managed node group with a specific instance profile
        
        Args:
            cluster_name: Name of the EKS cluster
            nodegroup_name: Name of the node group
            instance_types: List of EC2 instance types
            ami_type: AMI type for nodes
            capacity_type: Capacity type (ON_DEMAND or SPOT)
            scaling_config: Scaling configuration
            subnets: List of subnet IDs
            node_role_arn: IAM role ARN for nodes
            instance_profile: EC2 instance profile name
            
        Returns:
            Dict containing node group information
        """
        try:
            # Use defaults if not provided
            if instance_types is None:
                instance_types = ["t3.medium"]
            
            if scaling_config is None:
                scaling_config = {
                    "minSize": 1,
                    "maxSize": 3,
                    "desiredSize": 2
                }
            
            if node_role_arn is None:
                node_role_arn = self._create_or_get_nodegroup_role()
            
            if subnets is None:
                # Get cluster's subnets
                cluster_info = self.get_cluster_status(cluster_name)
                cluster_details = self.eks_client.describe_cluster(name=cluster_name)
                subnets = cluster_details['cluster']['resourcesVpcConfig']['subnetIds']
            
            # Create the node group
            response = self.eks_client.create_nodegroup(
                clusterName=cluster_name,
                nodegroupName=nodegroup_name,
                scalingConfig=scaling_config,
                instanceTypes=instance_types,
                amiType=ami_type,
                capacityType=capacity_type,
                nodeRole=node_role_arn,
                subnets=subnets,
                remoteAccess={
                    'ec2SshKey': None  # SSH key is optional
                },
                instanceProfile=instance_profile
            )
            
            return {
                'nodegroup_name': nodegroup_name,
                'cluster_name': cluster_name,
                'status': 'CREATING',
                'nodegroup_arn': response['nodegroup']['nodegroupArn'],
                'instance_types': instance_types,
                'ami_type': ami_type,
                'capacity_type': capacity_type,
                'scaling_config': scaling_config,
                'created_at': response['nodegroup']['createdAt'],
                'node_role_arn': node_role_arn,
                'instance_profile': instance_profile
            }
            
        except ClientError as e:
            raise Exception(f"Failed to create node group with instance profile: {e}")
    
    def update_nodegroup_scaling_config(self, cluster_name: str, nodegroup_name: str, 
                                       scaling_config: Dict) -> bool:
        """Update the scaling configuration of a node group"""
        try:
            # Get current node group configuration
            nodegroup = self.get_nodegroup_status(cluster_name, nodegroup_name)
            
            # Update scaling configuration
            response = self.eks_client.update_nodegroup_config(
                clusterName=cluster_name,
                nodegroupName=nodegroup_name,
                scalingConfig=scaling_config
            )
            
            print(f"✅ Node group '{nodegroup_name}' scaling configuration updated")
            return True
            
        except ClientError as e:
            print(f"❌ Error updating node group scaling configuration: {e}")
            return False
    
    def update_nodegroup_instance_type(self, cluster_name: str, nodegroup_name: str, 
                                      instance_types: List[str]) -> bool:
        """Update the instance type of a node group"""
        try:
            # Get current node group configuration
            nodegroup = self.get_nodegroup_status(cluster_name, nodegroup_name)
            
            # Update instance type
            response = self.eks_client.update_nodegroup_config(
                clusterName=cluster_name,
                nodegroupName=nodegroup_name,
                instanceTypes=instance_types
            )
            
            print(f"✅ Node group '{nodegroup_name}' instance type updated")
            return True
            
        except ClientError as e:
            print(f"❌ Error updating node group instance type: {e}")
            return False
    
    def delete_nodegroup(self, cluster_name: str, nodegroup_name: str) -> bool:
        """Delete an EKS managed node group"""
        try:
            self.eks_client.delete_nodegroup(
                clusterName=cluster_name,
                nodegroupName=nodegroup_name
            )
            print(f"✅ Node group '{nodegroup_name}' deletion initiated")
            return True
        except ClientError as e:
            print(f"❌ Error deleting node group '{nodegroup_name}': {e}")
            return False

    def get_cluster_status(self, cluster_name: str) -> Dict:
        """Get EKS cluster status"""
        try:
            response = self.eks_client.describe_cluster(name=cluster_name)
            cluster = response['cluster']
            
            return {
                'name': cluster['name'],
                'status': cluster['status'],
                'endpoint': cluster.get('endpoint', 'Not available'),
                'version': cluster['version'],
                'platform_version': cluster.get('platformVersion', 'Not available'),
                'created_at': cluster['createdAt'],
                'arn': cluster['arn']
            }
            
        except ClientError as e:
            raise Exception(f"Failed to get cluster status: {e}")
    
    def wait_for_cluster_active(self, cluster_name: str, timeout: int = 1800) -> bool:
        """Wait for EKS cluster to become active"""
        import time
        start_time = time.time()
        
        while time.time() - start_time < timeout:
            try:
                status = self.get_cluster_status(cluster_name)
                if status['status'] == 'ACTIVE':
                    return True
                elif status['status'] == 'FAILED':
                    raise Exception(f"Cluster creation failed")
                
                time.sleep(30)  # Check every 30 seconds
                
            except Exception as e:
                raise Exception(f"Error waiting for cluster: {e}")
        
        return False

class K8sClient:
    def __init__(self, namespace="default"):
        try:
            config.load_kube_config()  # Loads from ~/.kube/config
        except:
            config.load_incluster_config()  # For running inside a cluster

        self.namespace = namespace
        self.apps_v1 = client.AppsV1Api()
        self.core_v1 = client.CoreV1Api()

    # ======================
    # DEPLOYMENT OPERATIONS
    # ======================
    def create_deployment(self, name: str, image: str, replicas: int = 1, 
                         container_port: int = 80, env_vars: Optional[Dict[str, str]] = None,
                         labels: Optional[Dict[str, str]] = None, 
                         init_containers: Optional[List[Dict]] = None,
                         volume_mounts: Optional[List[Dict]] = None,
                         volumes: Optional[List[Dict]] = None) -> Optional[Any]:
        """Create a Kubernetes deployment
        
        Args:
            name: Deployment name
            image: Container image
            replicas: Number of replicas
            container_port: Container port
            env_vars: Environment variables
            labels: Labels for the deployment
            init_containers: List of init container specifications
            volume_mounts: List of volume mounts for the main container
            volumes: List of volumes for the pod
            
        Returns:
            Deployment object if successful, None otherwise
        """
        if labels is None:
            labels = {"app": name}
        
        # Environment variables
        env = []
        if env_vars:
            env = [client.V1EnvVar(name=k, value=v) for k, v in env_vars.items()]
        
        # Volume mounts for main container
        volume_mounts_obj = []
        if volume_mounts:
            for vm in volume_mounts:
                volume_mounts_obj.append(client.V1VolumeMount(
                    name=vm.get('name'),
                    mount_path=vm.get('mount_path'),
                    read_only=vm.get('read_only', False)
                ))
        
        # Main container
        container = client.V1Container(
            name=name,
            image=image,
            ports=[client.V1ContainerPort(container_port=container_port)],
            env=env if env else None,
            volume_mounts=volume_mounts_obj if volume_mounts_obj else None
        )

        # Init containers
        init_containers_obj = []
        if init_containers:
            for init_container in init_containers:
                init_env = []
                if init_container.get('env_vars'):
                    init_env = [client.V1EnvVar(name=k, value=v) 
                              for k, v in init_container['env_vars'].items()]
                
                init_volume_mounts = []
                if init_container.get('volume_mounts'):
                    for vm in init_container['volume_mounts']:
                        init_volume_mounts.append(client.V1VolumeMount(
                            name=vm.get('name'),
                            mount_path=vm.get('mount_path'),
                            read_only=vm.get('read_only', False)
                        ))
                
                init_containers_obj.append(client.V1Container(
                    name=init_container['name'],
                    image=init_container['image'],
                    command=init_container.get('command'),
                    args=init_container.get('args'),
                    env=init_env if init_env else None,
                    volume_mounts=init_volume_mounts if init_volume_mounts else None
                ))
        
        # Volumes
        volumes_obj = []
        if volumes:
            for volume in volumes:
                if volume.get('type') == 'pvc':
                    volumes_obj.append(client.V1Volume(
                        name=volume['name'],
                        persistent_volume_claim=client.V1PersistentVolumeClaimVolumeSource(
                            claim_name=volume['claim_name']
                        )
                    ))
                elif volume.get('type') == 'secret':
                    volumes_obj.append(client.V1Volume(
                        name=volume['name'],
                        secret=client.V1SecretVolumeSource(
                            secret_name=volume['secret_name']
                        )
                    ))
                elif volume.get('type') == 'configmap':
                    volumes_obj.append(client.V1Volume(
                        name=volume['name'],
                        config_map=client.V1ConfigMapVolumeSource(
                            name=volume['config_map_name']
                        )
                    ))
                elif volume.get('type') == 'empty_dir':
                    volumes_obj.append(client.V1Volume(
                        name=volume['name'],
                        empty_dir=client.V1EmptyDirVolumeSource()
                    ))

        template = client.V1PodTemplateSpec(
            metadata=client.V1ObjectMeta(labels=labels),
            spec=client.V1PodSpec(
                containers=[container],
                init_containers=init_containers_obj if init_containers_obj else None,
                volumes=volumes_obj if volumes_obj else None
            )
        )

        spec = client.V1DeploymentSpec(
            replicas=replicas,
            template=template,
            selector=client.V1LabelSelector(match_labels=labels)
        )

        deployment = client.V1Deployment(
            metadata=client.V1ObjectMeta(name=name, labels=labels),
            spec=spec
        )

        try:
            resp = self.apps_v1.create_namespaced_deployment(
                body=deployment,
                namespace=self.namespace
            )
            print(f"✅ Deployment '{name}' created successfully")
            return resp
        except ApiException as e:
            print(f"❌ Error creating deployment '{name}': {e}")
            return None

    def delete_deployment(self, name: str) -> bool:
        """Delete a Kubernetes deployment"""
        try:
            self.apps_v1.delete_namespaced_deployment(
                name=name,
                namespace=self.namespace
            )
            print(f"✅ Deployment '{name}' deleted successfully")
            return True
        except ApiException as e:
            print(f"❌ Error deleting deployment '{name}': {e}")
            return False

    def scale_deployment(self, name: str, replicas: int) -> bool:
        """Scale a deployment to the specified number of replicas"""
        try:
            # Get current deployment
            deployment = self.apps_v1.read_namespaced_deployment(
                name=name,
                namespace=self.namespace
            )
            
            # Update replicas
            deployment.spec.replicas = replicas
            
            # Apply the update
            self.apps_v1.patch_namespaced_deployment(
                name=name,
                namespace=self.namespace,
                body=deployment
            )
            print(f"✅ Deployment '{name}' scaled to {replicas} replicas")
            return True
        except ApiException as e:
            print(f"❌ Error scaling deployment '{name}': {e}")
            return False

    def rolling_update_deployment(self, name: str, image: str = None, 
                                 replicas: int = None, env_vars: Optional[Dict[str, str]] = None,
                                 labels: Optional[Dict[str, str]] = None,
                                 container_port: int = None) -> bool:
        """Perform a rolling update on a deployment
        
        Args:
            name: Deployment name
            image: New container image (optional)
            replicas: New replica count (optional)
            env_vars: New environment variables (optional)
            labels: New labels (optional)
            container_port: New container port (optional)
            
        Returns:
            True if successful, False otherwise
        """
        try:
            # Get current deployment
            deployment = self.apps_v1.read_namespaced_deployment(
                name=name,
                namespace=self.namespace
            )
            
            # Update image if provided
            if image:
                for container in deployment.spec.template.spec.containers:
                    if container.name == name:  # Update main container
                        container.image = image
                        break
            
            # Update replicas if provided
            if replicas is not None:
                deployment.spec.replicas = replicas
            
            # Update environment variables if provided
            if env_vars:
                for container in deployment.spec.template.spec.containers:
                    if container.name == name:  # Update main container
                        env = [client.V1EnvVar(name=k, value=v) for k, v in env_vars.items()]
                        container.env = env
                        break
            
            # Update labels if provided
            if labels:
                if deployment.spec.template.metadata.labels:
                    deployment.spec.template.metadata.labels.update(labels)
                else:
                    deployment.spec.template.metadata.labels = labels
                
                # Also update selector labels if they match
                if deployment.spec.selector.match_labels:
                    deployment.spec.selector.match_labels.update(labels)
            
            # Update container port if provided
            if container_port:
                for container in deployment.spec.template.spec.containers:
                    if container.name == name:  # Update main container
                        if container.ports:
                            container.ports[0].container_port = container_port
                        else:
                            container.ports = [client.V1ContainerPort(container_port=container_port)]
                        break
            
            # Apply the rolling update
            self.apps_v1.patch_namespaced_deployment(
                name=name,
                namespace=self.namespace,
                body=deployment
            )
            
            return True
        except ApiException as e:
            print(f"❌ Error performing rolling update on deployment '{name}': {e}")
            return False

    def wait_for_rollout_complete(self, name: str, timeout: int = 300) -> bool:
        """Wait for a deployment rollout to complete
        
        Args:
            name: Deployment name
            timeout: Timeout in seconds
            
        Returns:
            True if rollout completed successfully, False otherwise
        """
        try:
            start_time = time.time()
            while time.time() - start_time < timeout:
                deployment = self.apps_v1.read_namespaced_deployment(
                    name=name,
                    namespace=self.namespace
                )
                
                # Check if rollout is complete
                if (deployment.status.ready_replicas == deployment.spec.replicas and
                    deployment.status.updated_replicas == deployment.spec.replicas and
                    deployment.status.available_replicas == deployment.spec.replicas):
                    return True
                
                time.sleep(2)
            
            return False
        except ApiException as e:
            print(f"❌ Error waiting for rollout: {e}")
            return False

    def get_rollout_status(self, name: str) -> Dict[str, Any]:
        """Get deployment rollout status
        
        Args:
            name: Deployment name
            
        Returns:
            Dictionary with rollout status information
        """
        try:
            deployment = self.apps_v1.read_namespaced_deployment(
                name=name,
                namespace=self.namespace
            )
            
            return {
                'name': deployment.metadata.name,
                'desired_replicas': deployment.spec.replicas,
                'current_replicas': deployment.status.replicas or 0,
                'updated_replicas': deployment.status.updated_replicas or 0,
                'ready_replicas': deployment.status.ready_replicas or 0,
                'available_replicas': deployment.status.available_replicas or 0,
                'unavailable_replicas': deployment.status.unavailable_replicas or 0,
                'conditions': [
                    {
                        'type': condition.type,
                        'status': condition.status,
                        'reason': condition.reason,
                        'message': condition.message,
                        'last_update_time': condition.last_update_time
                    } for condition in (deployment.status.conditions or [])
                ]
            }
        except ApiException as e:
            print(f"❌ Error getting rollout status: {e}")
            return {}

    def list_deployments(self) -> List[Dict[str, Any]]:
        """List all deployments in the namespace"""
        try:
            deployments = self.apps_v1.list_namespaced_deployment(namespace=self.namespace)
            result = []
            for deployment in deployments.items:
                result.append({
                    'name': deployment.metadata.name,
                    'replicas': deployment.spec.replicas,
                    'ready_replicas': deployment.status.ready_replicas or 0,
                    'available_replicas': deployment.status.available_replicas or 0,
                    'created': deployment.metadata.creation_timestamp
                })
            return result
        except ApiException as e:
            print(f"❌ Error listing deployments: {e}")
            return []

    # ======================
    # POD OPERATIONS
    # ======================
    def create_pod(self, name: str, image: str, container_port: int = 80,
                   env_vars: Optional[Dict[str, str]] = None,
                   labels: Optional[Dict[str, str]] = None) -> Optional[Any]:
        """Create a simple pod"""
        if labels is None:
            labels = {"app": name}
        
        # Environment variables
        env = []
        if env_vars:
            env = [client.V1EnvVar(name=k, value=v) for k, v in env_vars.items()]
        
        container = client.V1Container(
            name=name,
            image=image,
            ports=[client.V1ContainerPort(container_port=container_port)],
            env=env if env else None
        )

        pod = client.V1Pod(
            metadata=client.V1ObjectMeta(name=name, labels=labels),
            spec=client.V1PodSpec(containers=[container])
        )

        try:
            resp = self.core_v1.create_namespaced_pod(
                body=pod,
                namespace=self.namespace
            )
            print(f"✅ Pod '{name}' created successfully")
            return resp
        except ApiException as e:
            print(f"❌ Error creating pod '{name}': {e}")
            return None

    def delete_pod(self, name: str) -> bool:
        """Delete a pod"""
        try:
            self.core_v1.delete_namespaced_pod(
                name=name,
                namespace=self.namespace
            )
            print(f"✅ Pod '{name}' deleted successfully")
            return True
        except ApiException as e:
            print(f"❌ Error deleting pod '{name}': {e}")
            return False

    def list_pods(self) -> List[Dict[str, Any]]:
        """List all pods in the namespace"""
        try:
            pods = self.core_v1.list_namespaced_pod(namespace=self.namespace)
            result = []
            for pod in pods.items:
                result.append({
                    'name': pod.metadata.name,
                    'phase': pod.status.phase,
                    'ready': self._is_pod_ready(pod),
                    'restarts': self._get_pod_restarts(pod),
                    'age': pod.metadata.creation_timestamp,
                    'node': pod.spec.node_name
                })
            return result
        except ApiException as e:
            print(f"❌ Error listing pods: {e}")
            return []

    def _is_pod_ready(self, pod) -> bool:
        """Check if a pod is ready"""
        if pod.status.conditions:
            for condition in pod.status.conditions:
                if condition.type == "Ready":
                    return condition.status == "True"
        return False

    def _get_pod_restarts(self, pod) -> int:
        """Get the number of restarts for a pod"""
        if pod.status.container_statuses:
            return sum(container.restart_count for container in pod.status.container_statuses)
        return 0

    def get_logs(self, pod_name: str, container_name: Optional[str] = None, 
                 tail_lines: Optional[int] = None) -> Optional[str]:
        """Get logs from a pod"""
        try:
            kwargs = {
                'name': pod_name,
                'namespace': self.namespace
            }
            if container_name:
                kwargs['container'] = container_name
            if tail_lines:
                kwargs['tail_lines'] = tail_lines
                
            logs = self.core_v1.read_namespaced_pod_log(**kwargs)
            print(f"📄 Logs from pod '{pod_name}':")
            print(logs)
            return logs
        except ApiException as e:
            print(f"❌ Error fetching logs from pod '{pod_name}': {e}")
            return None

    # ======================
    # SERVICE OPERATIONS
    # ======================
    def create_service(self, name: str, port: int, target_port: int, 
                      service_type: str = "ClusterIP", 
                      selector: Optional[Dict[str, str]] = None) -> Optional[Any]:
        """Create a Kubernetes service"""
        if selector is None:
            selector = {"app": name}
        
        service = client.V1Service(
            metadata=client.V1ObjectMeta(name=name),
            spec=client.V1ServiceSpec(
                selector=selector,
                ports=[client.V1ServicePort(
                    port=port,
                    target_port=target_port
                )],
                type=service_type
            )
        )

        try:
            resp = self.core_v1.create_namespaced_service(
                body=service,
                namespace=self.namespace
            )
            print(f"✅ Service '{name}' created successfully")
            return resp
        except ApiException as e:
            print(f"❌ Error creating service '{name}': {e}")
            return None

    def delete_service(self, name: str) -> bool:
        """Delete a service"""
        try:
            self.core_v1.delete_namespaced_service(
                name=name,
                namespace=self.namespace
            )
            print(f"✅ Service '{name}' deleted successfully")
            return True
        except ApiException as e:
            print(f"❌ Error deleting service '{name}': {e}")
            return False

    def list_services(self) -> List[Dict[str, Any]]:
        """List all services in the namespace"""
        try:
            services = self.core_v1.list_namespaced_service(namespace=self.namespace)
            result = []
            for service in services.items:
                result.append({
                    'name': service.metadata.name,
                    'type': service.spec.type,
                    'cluster_ip': service.spec.cluster_ip,
                    'external_ip': service.status.load_balancer.ingress[0].ip if (
                        service.status.load_balancer and 
                        service.status.load_balancer.ingress
                    ) else None,
                    'ports': [{'port': port.port, 'target_port': port.target_port} 
                             for port in service.spec.ports],
                    'created': service.metadata.creation_timestamp
                })
            return result
        except ApiException as e:
            print(f"❌ Error listing services: {e}")
            return []

    # ======================
    # EVENTS AND MONITORING
    # ======================
    def get_events(self, resource_name: Optional[str] = None) -> List[Dict[str, Any]]:
        """Get events from the namespace, optionally filtered by resource name"""
        try:
            events = self.core_v1.list_namespaced_event(namespace=self.namespace)
            result = []
            
            for event in events.items:
                if resource_name and event.involved_object.name != resource_name:
                    continue
                    
                result.append({
                    'name': event.metadata.name,
                    'type': event.type,
                    'reason': event.reason,
                    'message': event.message,
                    'resource': f"{event.involved_object.kind}/{event.involved_object.name}",
                    'first_timestamp': event.first_timestamp,
                    'last_timestamp': event.last_timestamp,
                    'count': event.count
                })
            
            return sorted(result, key=lambda x: x['last_timestamp'] or x['first_timestamp'], reverse=True)
        except ApiException as e:
            print(f"❌ Error fetching events: {e}")
            return []

    # ======================
    # RESOURCE DESCRIPTION
    # ======================
    def describe_pod(self, name: str) -> Optional[Dict[str, Any]]:
        """Get detailed information about a pod"""
        try:
            pod = self.core_v1.read_namespaced_pod(name=name, namespace=self.namespace)
            
            return {
                'metadata': {
                    'name': pod.metadata.name,
                    'namespace': pod.metadata.namespace,
                    'labels': pod.metadata.labels,
                    'annotations': pod.metadata.annotations,
                    'creation_timestamp': pod.metadata.creation_timestamp
                },
                'spec': {
                    'containers': [
                        {
                            'name': container.name,
                            'image': container.image,
                            'ports': [{'container_port': port.container_port} for port in container.ports] if container.ports else [],
                            'env': [{'name': env.name, 'value': env.value} for env in container.env] if container.env else []
                        }
                        for container in pod.spec.containers
                    ],
                    'restart_policy': pod.spec.restart_policy,
                    'node_name': pod.spec.node_name
                },
                'status': {
                    'phase': pod.status.phase,
                    'conditions': [
                        {
                            'type': condition.type,
                            'status': condition.status,
                            'reason': condition.reason,
                            'message': condition.message
                        }
                        for condition in pod.status.conditions
                    ] if pod.status.conditions else [],
                    'container_statuses': [
                        {
                            'name': status.name,
                            'ready': status.ready,
                            'restart_count': status.restart_count,
                            'state': str(status.state)
                        }
                        for status in pod.status.container_statuses
                    ] if pod.status.container_statuses else []
                }
            }
        except ApiException as e:
            print(f"❌ Error describing pod '{name}': {e}")
            return None

    def describe_deployment(self, name: str) -> Optional[Dict[str, Any]]:
        """Get detailed information about a deployment"""
        try:
            deployment = self.apps_v1.read_namespaced_deployment(name=name, namespace=self.namespace)
            
            return {
                'metadata': {
                    'name': deployment.metadata.name,
                    'namespace': deployment.metadata.namespace,
                    'labels': deployment.metadata.labels,
                    'annotations': deployment.metadata.annotations,
                    'creation_timestamp': deployment.metadata.creation_timestamp
                },
                'spec': {
                    'replicas': deployment.spec.replicas,
                    'selector': deployment.spec.selector.match_labels,
                    'template': {
                        'metadata': {
                            'labels': deployment.spec.template.metadata.labels
                        },
                        'spec': {
                            'containers': [
                                {
                                    'name': container.name,
                                    'image': container.image,
                                    'ports': [{'container_port': port.container_port} for port in container.ports] if container.ports else []
                                }
                                for container in deployment.spec.template.spec.containers
                            ]
                        }
                    }
                },
                'status': {
                    'replicas': deployment.status.replicas,
                    'ready_replicas': deployment.status.ready_replicas,
                    'available_replicas': deployment.status.available_replicas,
                    'unavailable_replicas': deployment.status.unavailable_replicas,
                    'conditions': [
                        {
                            'type': condition.type,
                            'status': condition.status,
                            'reason': condition.reason,
                            'message': condition.message
                        }
                        for condition in deployment.status.conditions
                    ] if deployment.status.conditions else []
                }
            }
        except ApiException as e:
            print(f"❌ Error describing deployment '{name}': {e}")
            return None

    def describe_service(self, name: str) -> Optional[Dict[str, Any]]:
        """Get detailed information about a service"""
        try:
            service = self.core_v1.read_namespaced_service(name=name, namespace=self.namespace)
            
            return {
                'metadata': {
                    'name': service.metadata.name,
                    'namespace': service.metadata.namespace,
                    'labels': service.metadata.labels,
                    'annotations': service.metadata.annotations,
                    'creation_timestamp': service.metadata.creation_timestamp
                },
                'spec': {
                    'type': service.spec.type,
                    'selector': service.spec.selector,
                    'ports': [
                        {
                            'port': port.port,
                            'target_port': port.target_port,
                            'protocol': port.protocol
                        }
                        for port in service.spec.ports
                    ],
                    'cluster_ip': service.spec.cluster_ip
                },
                'status': {
                    'load_balancer': {
                        'ingress': [
                            {'ip': ingress.ip, 'hostname': ingress.hostname}
                            for ingress in service.status.load_balancer.ingress
                        ] if service.status.load_balancer and service.status.load_balancer.ingress else []
                    }
                }
            }
        except ApiException as e:
            print(f"❌ Error describing service '{name}': {e}")
            return None

    # ======================
    # SECRET OPERATIONS
    # ======================
    def create_secret(self, name: str, data: Dict[str, str], 
                     secret_type: str = "Opaque", namespace: str = None) -> Optional[Any]:
        """Create a Kubernetes secret
        
        Args:
            name: Name of the secret
            data: Dictionary of key-value pairs for the secret
            secret_type: Type of secret (Opaque, kubernetes.io/tls, etc.)
            namespace: Namespace (uses default if not provided)
            
        Returns:
            Secret object if successful, None otherwise
        """
        try:
            ns = namespace or self.namespace
            
            # Encode data as base64
            encoded_data = {}
            for key, value in data.items():
                encoded_data[key] = base64.b64encode(value.encode()).decode()
            
            secret = client.V1Secret(
                metadata=client.V1ObjectMeta(name=name, namespace=ns),
                type=secret_type,
                data=encoded_data
            )
            
            result = self.core_v1.create_namespaced_secret(
                namespace=ns,
                body=secret
            )
            
            return result
            
        except ApiException as e:
            print(f"❌ Error creating secret: {e}")
            return None
    
    def get_secret(self, name: str, namespace: str = None) -> Optional[Dict]:
        """Get a Kubernetes secret
        
        Args:
            name: Name of the secret
            namespace: Namespace (uses default if not provided)
            
        Returns:
            Dictionary containing secret data
        """
        try:
            ns = namespace or self.namespace
            result = self.core_v1.read_namespaced_secret(name=name, namespace=ns)
            
            # Decode base64 data
            decoded_data = {}
            if result.data:
                for key, value in result.data.items():
                    decoded_data[key] = base64.b64decode(value).decode()
            
            return {
                'name': result.metadata.name,
                'namespace': result.metadata.namespace,
                'type': result.type,
                'data': decoded_data,
                'created_at': result.metadata.creation_timestamp
            }
            
        except ApiException as e:
            print(f"❌ Error getting secret: {e}")
            return None
    
    def delete_secret(self, name: str, namespace: str = None) -> bool:
        """Delete a Kubernetes secret"""
        try:
            ns = namespace or self.namespace
            self.core_v1.delete_namespaced_secret(name=name, namespace=ns)
            return True
        except ApiException as e:
            print(f"❌ Error deleting secret: {e}")
            return False
    
    def list_secrets(self, namespace: str = None) -> List[Dict]:
        """List all secrets in a namespace"""
        try:
            ns = namespace or self.namespace
            result = self.core_v1.list_namespaced_secret(namespace=ns)
            
            secrets = []
            for secret in result.items:
                secrets.append({
                    'name': secret.metadata.name,
                    'namespace': secret.metadata.namespace,
                    'type': secret.type,
                    'data_keys': list(secret.data.keys()) if secret.data else [],
                    'created_at': secret.metadata.creation_timestamp
                })
            
            return secrets
            
        except ApiException as e:
            print(f"❌ Error listing secrets: {e}")
            return []

    # ======================
    # PVC OPERATIONS
    # ======================
    def create_pvc(self, name: str, size: str, access_modes: List[str] = None,
                   storage_class: str = None, namespace: str = None) -> Optional[Any]:
        """Create a Persistent Volume Claim
        
        Args:
            name: Name of the PVC
            size: Size of the volume (e.g., '10Gi', '100Mi')
            access_modes: List of access modes (default: ['ReadWriteOnce'])
            storage_class: Storage class name
            namespace: Namespace (uses default if not provided)
            
        Returns:
            PVC object if successful, None otherwise
        """
        try:
            ns = namespace or self.namespace
            
            if access_modes is None:
                access_modes = ['ReadWriteOnce']
            
            # Create PVC specification
            pvc_spec = client.V1PersistentVolumeClaimSpec(
                access_modes=access_modes,
                resources=client.V1ResourceRequirements(
                    requests={'storage': size}
                )
            )
            
            if storage_class:
                pvc_spec.storage_class_name = storage_class
            
            pvc = client.V1PersistentVolumeClaim(
                metadata=client.V1ObjectMeta(name=name, namespace=ns),
                spec=pvc_spec
            )
            
            result = self.core_v1.create_namespaced_persistent_volume_claim(
                namespace=ns,
                body=pvc
            )
            
            return result
            
        except ApiException as e:
            print(f"❌ Error creating PVC: {e}")
            return None
    
    def get_pvc(self, name: str, namespace: str = None) -> Optional[Dict]:
        """Get a Persistent Volume Claim"""
        try:
            ns = namespace or self.namespace
            result = self.core_v1.read_namespaced_persistent_volume_claim(name=name, namespace=ns)
            
            return {
                'name': result.metadata.name,
                'namespace': result.metadata.namespace,
                'status': result.status.phase,
                'volume_name': result.spec.volume_name,
                'access_modes': result.spec.access_modes,
                'storage_class': result.spec.storage_class_name,
                'size': result.spec.resources.requests.get('storage', 'Unknown'),
                'created_at': result.metadata.creation_timestamp
            }
            
        except ApiException as e:
            print(f"❌ Error getting PVC: {e}")
            return None
    
    def delete_pvc(self, name: str, namespace: str = None) -> bool:
        """Delete a Persistent Volume Claim"""
        try:
            ns = namespace or self.namespace
            self.core_v1.delete_namespaced_persistent_volume_claim(name=name, namespace=ns)
            return True
        except ApiException as e:
            print(f"❌ Error deleting PVC: {e}")
            return False
    
    def list_pvcs(self, namespace: str = None) -> List[Dict]:
        """List all PVCs in a namespace"""
        try:
            ns = namespace or self.namespace
            result = self.core_v1.list_namespaced_persistent_volume_claim(namespace=ns)
            
            pvcs = []
            for pvc in result.items:
                pvcs.append({
                    'name': pvc.metadata.name,
                    'namespace': pvc.metadata.namespace,
                    'status': pvc.status.phase,
                    'volume_name': pvc.spec.volume_name,
                    'access_modes': pvc.spec.access_modes,
                    'storage_class': pvc.spec.storage_class_name,
                    'size': pvc.spec.resources.requests.get('storage', 'Unknown'),
                    'created_at': pvc.metadata.creation_timestamp
                })
            
            return pvcs
            
        except ApiException as e:
            print(f"❌ Error listing PVCs: {e}")
            return []

    # ======================
    # SERVICE URL OPERATIONS
    # ======================
    def get_service_url(self, name: str, namespace: str = None) -> Optional[Dict]:
        """Get service URL, including AWS ELB URLs for LoadBalancer services
        
        Args:
            name: Name of the service
            namespace: Namespace (uses default if not provided)
            
        Returns:
            Dictionary containing service URL information
        """
        try:
            ns = namespace or self.namespace
            service = self.core_v1.read_namespaced_service(name=name, namespace=ns)
            
            service_type = service.spec.type
            ports = []
            for port in service.spec.ports:
                ports.append({
                    'port': port.port,
                    'target_port': port.target_port,
                    'protocol': port.protocol,
                    'name': port.name
                })
            
            result = {
                'name': name,
                'namespace': ns,
                'type': service_type,
                'ports': ports,
                'cluster_ip': service.spec.cluster_ip
            }
            
            if service_type == 'LoadBalancer':
                # Check for AWS ELB
                ingress = service.status.load_balancer.ingress
                if ingress:
                    for ing in ingress:
                        if ing.hostname:  # AWS ELB uses hostname
                            result['external_url'] = f"http://{ing.hostname}"
                            result['external_hostname'] = ing.hostname
                            
                            # Check if it's an AWS ELB
                            if 'elb.amazonaws.com' in ing.hostname:
                                result['aws_elb'] = True
                                result['elb_dns_name'] = ing.hostname
                        elif ing.ip:  # Some cloud providers use IP
                            result['external_url'] = f"http://{ing.ip}"
                            result['external_ip'] = ing.ip
                
                # If no ingress yet, service might still be provisioning
                if not ingress:
                    result['status'] = 'Provisioning LoadBalancer...'
            
            elif service_type == 'NodePort':
                # For NodePort, we need to get node IPs
                nodes = self.core_v1.list_node()
                if nodes.items:
                    node_ip = None
                    for node in nodes.items:
                        for address in node.status.addresses:
                            if address.type == 'ExternalIP':
                                node_ip = address.address
                                break
                        if node_ip:
                            break
                    
                    if node_ip:
                        for port in service.spec.ports:
                            if port.node_port:
                                result['external_url'] = f"http://{node_ip}:{port.node_port}"
                                result['node_ip'] = node_ip
                                result['node_port'] = port.node_port
            
            return result
            
        except ApiException as e:
            print(f"❌ Error getting service URL: {e}")
            return None

    # ======================
    # UTILITY METHODS
    # ======================
    def get_namespace_resources(self) -> Dict[str, int]:
        """Get a summary of resources in the namespace"""
        try:
            pods = len(self.core_v1.list_namespaced_pod(namespace=self.namespace).items)
            deployments = len(self.apps_v1.list_namespaced_deployment(namespace=self.namespace).items)
            services = len(self.core_v1.list_namespaced_service(namespace=self.namespace).items)
            
            return {
                'pods': pods,
                'deployments': deployments,
                'services': services
            }
        except ApiException as e:
            print(f"❌ Error getting namespace resources: {e}")
            return {}

    def wait_for_deployment_ready(self, name: str, timeout: int = 300) -> bool:
        """Wait for a deployment to be ready"""
        import time
        start_time = time.time()
        
        while time.time() - start_time < timeout:
            try:
                deployment = self.apps_v1.read_namespaced_deployment(name=name, namespace=self.namespace)
                if (deployment.status.ready_replicas == deployment.spec.replicas and 
                    deployment.status.ready_replicas > 0):
                    print(f"✅ Deployment '{name}' is ready")
                    return True
                    
                print(f"⏳ Waiting for deployment '{name}' to be ready... ({deployment.status.ready_replicas or 0}/{deployment.spec.replicas})")
                time.sleep(5)
                
            except ApiException as e:
                print(f"❌ Error checking deployment status: {e}")
                return False
        
        print(f"❌ Timeout waiting for deployment '{name}' to be ready")
        return False

    # ======================
    # MONITORING OPERATIONS
    # ======================
    
    def _check_helm_available(self) -> bool:
        """Check if Helm is available in the system"""
        import subprocess
        try:
            result = subprocess.run(['helm', 'version', '--short'], 
                                  capture_output=True, text=True, timeout=10)
            return result.returncode == 0
        except (subprocess.TimeoutExpired, FileNotFoundError):
            return False
    
    def _install_helm_automatically(self) -> Dict[str, Any]:
        """Automatically install Helm based on the operating system"""
        import subprocess
        import platform
        import os
        import tempfile
        import urllib.request
        
        result = {
            'success': False,
            'method': None,
            'error': None,
            'message': None
        }
        
        system = platform.system().lower()
        print(f"🔧 Attempting to install Helm on {system}...")
        
        try:
            if system == 'windows':
                # Try to install via Chocolatey first, then Scoop, then direct download
                methods = [
                    {
                        'name': 'Chocolatey',
                        'check_cmd': ['choco', '--version'],
                        'install_cmd': ['choco', 'install', 'kubernetes-helm', '-y']
                    },
                    {
                        'name': 'Scoop',
                        'check_cmd': ['scoop', '--version'],
                        'install_cmd': ['scoop', 'install', 'helm']
                    }
                ]
                
                for method in methods:
                    try:
                        # Check if package manager is available
                        subprocess.run(method['check_cmd'], capture_output=True, check=True, timeout=10)
                        print(f"📦 Installing Helm via {method['name']}...")
                        
                        # Install Helm
                        install_result = subprocess.run(
                            method['install_cmd'], 
                            capture_output=True, 
                            text=True, 
                            timeout=300  # 5 minutes timeout
                        )
                        
                        if install_result.returncode == 0:
                            result['success'] = True
                            result['method'] = method['name']
                            result['message'] = f"Helm installed successfully via {method['name']}"
                            print(f"✅ {result['message']}")
                            return result
                        else:
                            print(f"⚠️  {method['name']} installation failed: {install_result.stderr}")
                            
                    except (subprocess.TimeoutExpired, subprocess.CalledProcessError, FileNotFoundError):
                        print(f"⚠️  {method['name']} not available")
                        continue
                
                # If package managers fail, try direct download
                print("📦 Attempting direct download installation...")
                result = self._install_helm_direct_download()
                
            elif system == 'linux':
                # Try different Linux package managers and methods
                methods = [
                    {
                        'name': 'Snap',
                        'check_cmd': ['snap', '--version'],
                        'install_cmd': ['sudo', 'snap', 'install', 'helm', '--classic']
                    },
                    {
                        'name': 'Script',
                        'script_url': 'https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3'
                    }
                ]
                
                # Try snap first
                try:
                    subprocess.run(['snap', '--version'], capture_output=True, check=True, timeout=10)
                    print("📦 Installing Helm via Snap...")
                    
                    install_result = subprocess.run(
                        ['sudo', 'snap', 'install', 'helm', '--classic'],
                        capture_output=True,
                        text=True,
                        timeout=300
                    )
                    
                    if install_result.returncode == 0:
                        result['success'] = True
                        result['method'] = 'Snap'
                        result['message'] = "Helm installed successfully via Snap"
                        print(f"✅ {result['message']}")
                        return result
                        
                except (subprocess.TimeoutExpired, subprocess.CalledProcessError, FileNotFoundError):
                    print("⚠️  Snap not available")
                
                # Try installation script
                print("📦 Installing Helm via official installation script...")
                try:
                    script_result = subprocess.run([
                        'bash', '-c', 
                        'curl -fsSL https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash'
                    ], capture_output=True, text=True, timeout=300)
                    
                    if script_result.returncode == 0:
                        result['success'] = True
                        result['method'] = 'Installation Script'
                        result['message'] = "Helm installed successfully via installation script"
                        print(f"✅ {result['message']}")
                        return result
                    else:
                        print(f"⚠️  Script installation failed: {script_result.stderr}")
                        
                except (subprocess.TimeoutExpired, subprocess.CalledProcessError):
                    print("⚠️  Script installation failed")
                
            elif system == 'darwin':  # macOS
                # Try Homebrew
                try:
                    subprocess.run(['brew', '--version'], capture_output=True, check=True, timeout=10)
                    print("📦 Installing Helm via Homebrew...")
                    
                    install_result = subprocess.run(
                        ['brew', 'install', 'helm'],
                        capture_output=True,
                        text=True,
                        timeout=300
                    )
                    
                    if install_result.returncode == 0:
                        result['success'] = True
                        result['method'] = 'Homebrew'
                        result['message'] = "Helm installed successfully via Homebrew"
                        print(f"✅ {result['message']}")
                        return result
                        
                except (subprocess.TimeoutExpired, subprocess.CalledProcessError, FileNotFoundError):
                    print("⚠️  Homebrew not available, trying installation script...")
                    
                    # Fallback to installation script
                    try:
                        script_result = subprocess.run([
                            'bash', '-c',
                            'curl -fsSL https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash'
                        ], capture_output=True, text=True, timeout=300)
                        
                        if script_result.returncode == 0:
                            result['success'] = True
                            result['method'] = 'Installation Script'
                            result['message'] = "Helm installed successfully via installation script"
                            print(f"✅ {result['message']}")
                            return result
                            
                    except (subprocess.TimeoutExpired, subprocess.CalledProcessError):
                        print("⚠️  Script installation failed")
            
            # If we reach here, all methods failed
            result['error'] = f"Could not install Helm automatically on {system}"
            print(f"❌ {result['error']}")
            
        except Exception as e:
            result['error'] = f"Error during Helm installation: {str(e)}"
            print(f"❌ {result['error']}")
        
        return result
    
    def _install_helm_direct_download(self) -> Dict[str, Any]:
        """Install Helm via direct download (Windows fallback)"""
        import subprocess
        import tempfile
        import zipfile
        import urllib.request
        import os
        import platform
        
        result = {
            'success': False,
            'method': 'Direct Download',
            'error': None,
            'message': None
        }
        
        try:
            # Determine architecture
            arch = platform.machine().lower()
            if arch == 'amd64' or arch == 'x86_64':
                arch = 'amd64'
            elif arch == 'arm64':
                arch = 'arm64'
            else:
                arch = 'amd64'  # Default fallback
            
            # Download URL for latest Helm
            download_url = f"https://get.helm.sh/helm-v3.13.0-windows-{arch}.zip"
            
            with tempfile.TemporaryDirectory() as temp_dir:
                zip_path = os.path.join(temp_dir, "helm.zip")
                
                print(f"📥 Downloading Helm from {download_url}...")
                urllib.request.urlretrieve(download_url, zip_path)
                
                print("📦 Extracting Helm...")
                with zipfile.ZipFile(zip_path, 'r') as zip_ref:
                    zip_ref.extractall(temp_dir)
                
                # Find helm.exe
                helm_exe = None
                for root, dirs, files in os.walk(temp_dir):
                    if 'helm.exe' in files:
                        helm_exe = os.path.join(root, 'helm.exe')
                        break
                
                if helm_exe:
                    # Try to move to a directory in PATH
                    target_locations = [
                        os.path.expanduser("~/bin"),
                        "C:\\Program Files\\Helm",
                        os.path.join(os.environ.get('LOCALAPPDATA', ''), 'Microsoft', 'WindowsApps')
                    ]
                    
                    for target_dir in target_locations:
                        try:
                            os.makedirs(target_dir, exist_ok=True)
                            target_path = os.path.join(target_dir, 'helm.exe')
                            
                            import shutil
                            shutil.copy2(helm_exe, target_path)
                            
                            # Test if it works
                            test_result = subprocess.run([target_path, 'version'], 
                                                       capture_output=True, timeout=10)
                            if test_result.returncode == 0:
                                result['success'] = True
                                result['message'] = f"Helm installed to {target_path}"
                                print(f"✅ {result['message']}")
                                print(f"💡 Please add {target_dir} to your PATH if not already present")
                                return result
                                
                        except (OSError, subprocess.TimeoutExpired):
                            continue
                    
                    result['error'] = "Downloaded Helm but could not install to PATH"
                else:
                    result['error'] = "Could not find helm.exe in downloaded archive"
                    
        except Exception as e:
            result['error'] = f"Direct download failed: {str(e)}"
        
        return result
    
    def _install_kube_state_metrics(self, namespace: str) -> Dict[str, Any]:
        """Install kube-state-metrics using Helm if available, or manual YAML if not"""
        import subprocess
        
        result = {
            'installed': False,
            'method': None,
            'error': None
        }
        
        # Check if kube-state-metrics is already running
        try:
            deployments = self.apps_v1.list_deployment_for_all_namespaces()
            for deployment in deployments.items:
                if 'kube-state-metrics' in deployment.metadata.name:
                    print(f"✅ kube-state-metrics already deployed in namespace: {deployment.metadata.namespace}")
                    result['installed'] = True
                    result['method'] = 'existing'
                    return result
        except Exception as e:
            print(f"⚠️  Warning: Could not check existing deployments: {e}")
        
        # Try Helm installation first
        if self._check_helm_available():
            try:
                print("📦 Installing kube-state-metrics using Helm...")
                
                # Add prometheus-community repo if not exists
                subprocess.run(['helm', 'repo', 'add', 'prometheus-community', 
                              'https://prometheus-community.github.io/helm-charts'], 
                              capture_output=True, text=True, timeout=30)
                
                # Update repo
                subprocess.run(['helm', 'repo', 'update'], 
                              capture_output=True, text=True, timeout=30)
                
                # Install kube-state-metrics
                helm_cmd = [
                    'helm', 'install', 'kube-state-metrics',
                    'prometheus-community/kube-state-metrics',
                    '--namespace', namespace,
                    '--create-namespace',
                    '--set', 'service.port=8080',
                    '--set', 'service.targetPort=8080'
                ]
                
                helm_result = subprocess.run(helm_cmd, capture_output=True, text=True, timeout=120)
                
                if helm_result.returncode == 0:
                    print("✅ kube-state-metrics installed successfully via Helm")
                    result['installed'] = True
                    result['method'] = 'helm'
                    return result
                else:
                    print(f"⚠️  Helm installation failed: {helm_result.stderr}")
                    
            except subprocess.TimeoutExpired:
                print("⚠️  Helm installation timed out, falling back to manual installation")
            except Exception as e:
                print(f"⚠️  Helm installation failed: {e}, falling back to manual installation")
        
        # Fallback to manual YAML installation
        try:
            print("📦 Installing kube-state-metrics using manual YAML...")
            
            # Create ServiceAccount
            service_account = client.V1ServiceAccount(
                metadata=client.V1ObjectMeta(
                    name="kube-state-metrics",
                    namespace=namespace
                )
            )
            
            try:
                self.core_v1.create_namespaced_service_account(namespace=namespace, body=service_account)
                print("✅ Created ServiceAccount for kube-state-metrics")
            except ApiException as e:
                if e.status == 409:
                    print("⚠️  ServiceAccount already exists")
                else:
                    raise e
            
            # Create ClusterRole
            cluster_role = client.V1ClusterRole(
                metadata=client.V1ObjectMeta(name="kube-state-metrics"),
                rules=[
                    client.V1PolicyRule(
                        api_groups=[""],
                        resources=["configmaps", "secrets", "nodes", "pods", "services", 
                                 "resourcequotas", "replicationcontrollers", "limitranges", 
                                 "persistentvolumeclaims", "persistentvolumes", "namespaces", "endpoints"],
                        verbs=["list", "watch"]
                    ),
                    client.V1PolicyRule(
                        api_groups=["apps"],
                        resources=["statefulsets", "daemonsets", "deployments", "replicasets"],
                        verbs=["list", "watch"]
                    ),
                    client.V1PolicyRule(
                        api_groups=["batch"],
                        resources=["cronjobs", "jobs"],
                        verbs=["list", "watch"]
                    ),
                    client.V1PolicyRule(
                        api_groups=["autoscaling"],
                        resources=["horizontalpodautoscalers"],
                        verbs=["list", "watch"]
                    ),
                    client.V1PolicyRule(
                        api_groups=["authentication.k8s.io"],
                        resources=["tokenreviews"],
                        verbs=["create"]
                    ),
                    client.V1PolicyRule(
                        api_groups=["authorization.k8s.io"],
                        resources=["subjectaccessreviews"],
                        verbs=["create"]
                    ),
                    client.V1PolicyRule(
                        api_groups=["policy"],
                        resources=["poddisruptionbudgets"],
                        verbs=["list", "watch"]
                    ),
                    client.V1PolicyRule(
                        api_groups=["certificates.k8s.io"],
                        resources=["certificatesigningrequests"],
                        verbs=["list", "watch"]
                    ),
                    client.V1PolicyRule(
                        api_groups=["storage.k8s.io"],
                        resources=["storageclasses", "volumeattachments"],
                        verbs=["list", "watch"]
                    ),
                    client.V1PolicyRule(
                        api_groups=["admissionregistration.k8s.io"],
                        resources=["mutatingwebhookconfigurations", "validatingwebhookconfigurations"],
                        verbs=["list", "watch"]
                    ),
                    client.V1PolicyRule(
                        api_groups=["networking.k8s.io"],
                        resources=["networkpolicies", "ingresses"],
                        verbs=["list", "watch"]
                    ),
                    client.V1PolicyRule(
                        api_groups=["coordination.k8s.io"],
                        resources=["leases"],
                        verbs=["list", "watch"]
                    )
                ]
            )
            
            # Create ClusterRole
            rbac_v1 = client.RbacAuthorizationV1Api()
            try:
                rbac_v1.create_cluster_role(body=cluster_role)
                print("✅ Created ClusterRole for kube-state-metrics")
            except ApiException as e:
                if e.status == 409:
                    print("⚠️  ClusterRole already exists")
                else:
                    raise e
            
            # Create ClusterRoleBinding
            # Create subject with version compatibility
            try:
                # Try V1Subject first (older versions)
                subject = client.V1Subject(
                    kind="ServiceAccount",
                    name="kube-state-metrics",
                    namespace=namespace
                )
            except AttributeError:
                # Try RbacV1Subject (newer versions)
                try:
                    subject = client.RbacV1Subject(
                        kind="ServiceAccount",
                        name="kube-state-metrics",
                        namespace=namespace
                    )
                except AttributeError:
                    # Manual construction as fallback
                    subject = {
                        'kind': 'ServiceAccount',
                        'name': 'kube-state-metrics',
                        'namespace': namespace
                    }
            
            cluster_role_binding = client.V1ClusterRoleBinding(
                metadata=client.V1ObjectMeta(name="kube-state-metrics"),
                subjects=[subject],
                role_ref=client.V1RoleRef(
                    kind="ClusterRole",
                    name="kube-state-metrics",
                    api_group="rbac.authorization.k8s.io"
                )
            )
            
            try:
                rbac_v1.create_cluster_role_binding(body=cluster_role_binding)
                print("✅ Created ClusterRoleBinding for kube-state-metrics")
            except ApiException as e:
                if e.status == 409:
                    print("⚠️  ClusterRoleBinding already exists")
                else:
                    raise e
            
            # Create Deployment
            deployment = client.V1Deployment(
                metadata=client.V1ObjectMeta(
                    name="kube-state-metrics",
                    namespace=namespace,
                    labels={"app": "kube-state-metrics"}
                ),
                spec=client.V1DeploymentSpec(
                    replicas=1,
                    selector=client.V1LabelSelector(
                        match_labels={"app": "kube-state-metrics"}
                    ),
                    template=client.V1PodTemplateSpec(
                        metadata=client.V1ObjectMeta(
                            labels={"app": "kube-state-metrics"}
                        ),
                        spec=client.V1PodSpec(
                            service_account_name="kube-state-metrics",
                            containers=[
                                client.V1Container(
                                    name="kube-state-metrics",
                                    image="registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.10.1",
                                    ports=[
                                        client.V1ContainerPort(
                                            name="http-metrics",
                                            container_port=8080,
                                            protocol="TCP"
                                        ),
                                        client.V1ContainerPort(
                                            name="telemetry",
                                            container_port=8081,
                                            protocol="TCP"
                                        )
                                    ],
                                    liveness_probe=client.V1Probe(
                                        http_get=client.V1HTTPGetAction(
                                            path="/healthz",
                                            port=8080
                                        ),
                                        initial_delay_seconds=5,
                                        timeout_seconds=5
                                    ),
                                    readiness_probe=client.V1Probe(
                                        http_get=client.V1HTTPGetAction(
                                            path="/",
                                            port=8081
                                        ),
                                        initial_delay_seconds=5,
                                        timeout_seconds=5
                                    ),
                                    security_context=client.V1SecurityContext(
                                        allow_privilege_escalation=False,
                                        read_only_root_filesystem=True,
                                        run_as_non_root=True,
                                        run_as_user=65534
                                    )
                                )
                            ]
                        )
                    )
                )
            )
            
            try:
                self.apps_v1.create_namespaced_deployment(namespace=namespace, body=deployment)
                print("✅ Created Deployment for kube-state-metrics")
            except ApiException as e:
                if e.status == 409:
                    print("⚠️  Deployment already exists")
                else:
                    raise e
            
            # Create Service
            service = client.V1Service(
                metadata=client.V1ObjectMeta(
                    name="kube-state-metrics",
                    namespace=namespace,
                    labels={"app": "kube-state-metrics"}
                ),
                spec=client.V1ServiceSpec(
                    selector={"app": "kube-state-metrics"},
                    ports=[
                        client.V1ServicePort(
                            name="http-metrics",
                            port=8080,
                            target_port=8080,
                            protocol="TCP"
                        ),
                        client.V1ServicePort(
                            name="telemetry",
                            port=8081,
                            target_port=8081,
                            protocol="TCP"
                        )
                    ],
                    type="ClusterIP"
                )
            )
            
            try:
                self.core_v1.create_namespaced_service(namespace=namespace, body=service)
                print("✅ Created Service for kube-state-metrics")
            except ApiException as e:
                if e.status == 409:
                    print("⚠️  Service already exists")
                else:
                    raise e
            
            result['installed'] = True
            result['method'] = 'manual'
            print("✅ kube-state-metrics installed successfully via manual YAML")
            
        except Exception as e:
            result['error'] = str(e)
            print(f"❌ Failed to install kube-state-metrics: {e}")
        
        return result
    
    def setup_monitoring(self, namespace: str = "monitoring", 
                        grafana_service_type: str = "NodePort",
                        import_dashboard: bool = True,
                        wait_for_ready: bool = True,
                        install_kube_state_metrics: bool = True) -> Dict[str, Any]:
        """Setup complete monitoring stack with Prometheus and Grafana
        
        Args:
            namespace: Namespace to deploy monitoring stack
            grafana_service_type: Service type for Grafana (NodePort, LoadBalancer, ClusterIP)
            import_dashboard: Whether to import default Kubernetes dashboard
            wait_for_ready: Whether to wait for deployments to be ready
            install_kube_state_metrics: Whether to install kube-state-metrics for cluster metrics
            
        Returns:
            Dictionary with deployment info, URLs, and credentials
        """
        result = {
            'namespace': namespace,
            'prometheus': {},
            'grafana': {},
            'kube_state_metrics': {},
            'success': False,
            'error': None
        }
        
        try:
            # Create monitoring namespace
            self._create_monitoring_namespace(namespace)
            
            # Install kube-state-metrics if requested
            if install_kube_state_metrics:
                print("📊 Installing kube-state-metrics for cluster metrics...")
                ksm_result = self._install_kube_state_metrics(namespace)
                result['kube_state_metrics'] = ksm_result
            
            # Deploy Prometheus
            prometheus_result = self._deploy_prometheus(namespace)
            result['prometheus'] = prometheus_result
            
            # Deploy Grafana
            grafana_result = self._deploy_grafana(namespace, grafana_service_type)
            result['grafana'] = grafana_result
            
            if wait_for_ready:
                # Wait for deployments to be ready
                if self._wait_for_monitoring_ready(namespace):
                    # Configure Grafana data source and dashboard
                    if self._configure_grafana(namespace, import_dashboard):
                        result['success'] = True
                        result['grafana_configured'] = True
                    else:
                        result['success'] = True  # Still successful even if config fails
                        result['grafana_configured'] = False
                        result['warning'] = "Grafana deployed but automatic configuration failed"
                else:
                    result['error'] = "Monitoring deployments failed to become ready"
            else:
                result['success'] = True
            
            return result
            
        except Exception as e:
            result['error'] = str(e)
            return result

    def _create_monitoring_namespace(self, namespace: str) -> bool:
        """Create monitoring namespace if it doesn't exist"""
        try:
            self.core_v1.read_namespace(name=namespace)
            print(f"✅ Namespace '{namespace}' already exists")
            return True
        except ApiException as e:
            if e.status == 404:
                # Create namespace
                namespace_obj = client.V1Namespace(
                    metadata=client.V1ObjectMeta(name=namespace)
                )
                self.core_v1.create_namespace(body=namespace_obj)
                print(f"✅ Created namespace '{namespace}'")
                return True
            else:
                raise e

    def _deploy_prometheus(self, namespace: str) -> Dict[str, Any]:
        """Deploy Prometheus to the cluster"""
        result = {'deployed': False, 'service_name': 'prometheus-service'}
        
        # Prometheus ConfigMap with kube-state-metrics support
        prometheus_config = """
global:
  scrape_interval: 15s
scrape_configs:
  - job_name: 'kubernetes-apiservers'
    kubernetes_sd_configs:
    - role: endpoints
    scheme: https
    tls_config:
      ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
    bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
    relabel_configs:
    - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
      action: keep
      regex: default;kubernetes;https

  - job_name: 'kubernetes-nodes'
    kubernetes_sd_configs:
    - role: node
    scheme: https
    tls_config:
      ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
    bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
    relabel_configs:
    - action: labelmap
      regex: __meta_kubernetes_node_label_(.+)

  - job_name: 'kubernetes-cadvisor'
    kubernetes_sd_configs:
    - role: node
    scheme: https
    tls_config:
      ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
    bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
    relabel_configs:
    - action: labelmap
      regex: __meta_kubernetes_node_label_(.+)
    - target_label: __address__
      replacement: kubernetes.default.svc:443
    - source_labels: [__meta_kubernetes_node_name]
      regex: (.+)
      target_label: __metrics_path__
      replacement: /api/v1/nodes/$1/proxy/metrics/cadvisor

  - job_name: 'kube-state-metrics'
    static_configs:
    - targets: ['kube-state-metrics.{}:8080']
    metrics_path: /metrics
    scrape_interval: 30s

  - job_name: 'kubernetes-pods'
    kubernetes_sd_configs:
    - role: pod
    relabel_configs:
    - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
      action: keep
      regex: true
    - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
      action: replace
      target_label: __metrics_path__
      regex: (.+)
    - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
      action: replace
      regex: ([^:]+)(?::\\d+)?;(\\d+)
      replacement: $1:$2
      target_label: __address__
    - action: labelmap
      regex: __meta_kubernetes_pod_label_(.+)
    - source_labels: [__meta_kubernetes_namespace]
      action: replace
      target_label: kubernetes_namespace
    - source_labels: [__meta_kubernetes_pod_name]
      action: replace
      target_label: kubernetes_pod_name

  - job_name: 'kubernetes-service-endpoints'
    kubernetes_sd_configs:
    - role: endpoints
    relabel_configs:
    - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
      action: keep
      regex: true
    - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
      action: replace
      target_label: __scheme__
      regex: (https?)
    - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
      action: replace
      target_label: __metrics_path__
      regex: (.+)
    - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
      action: replace
      target_label: __address__
      regex: ([^:]+)(?::\\d+)?;(\\d+)
      replacement: $1:$2
    - action: labelmap
      regex: __meta_kubernetes_service_label_(.+)
    - source_labels: [__meta_kubernetes_namespace]
      action: replace
      target_label: kubernetes_namespace
    - source_labels: [__meta_kubernetes_service_name]
      action: replace
      target_label: kubernetes_name
""".format(namespace)
        
        # Create ConfigMap
        configmap = client.V1ConfigMap(
            metadata=client.V1ObjectMeta(name="prometheus-config", namespace=namespace),
            data={"prometheus.yml": prometheus_config}
        )
        
        try:
            self.core_v1.create_namespaced_config_map(namespace=namespace, body=configmap)
        except ApiException as e:
            if e.status == 409:  # Already exists
                self.core_v1.replace_namespaced_config_map(
                    name="prometheus-config", namespace=namespace, body=configmap
                )
        
        # Create ServiceAccount and RBAC
        self._create_prometheus_rbac(namespace)
        
        # Prometheus Deployment
        deployment = client.V1Deployment(
            metadata=client.V1ObjectMeta(name="prometheus", namespace=namespace),
            spec=client.V1DeploymentSpec(
                replicas=1,
                selector=client.V1LabelSelector(match_labels={"app": "prometheus"}),
                template=client.V1PodTemplateSpec(
                    metadata=client.V1ObjectMeta(labels={"app": "prometheus"}),
                    spec=client.V1PodSpec(
                        service_account_name="prometheus",
                        containers=[
                            client.V1Container(
                                name="prometheus",
                                image="prom/prometheus:latest",
                                ports=[client.V1ContainerPort(container_port=9090)],
                                args=[
                                    "--config.file=/etc/prometheus/prometheus.yml",
                                    "--storage.tsdb.path=/prometheus/",
                                    "--web.console.libraries=/etc/prometheus/console_libraries",
                                    "--web.console.templates=/etc/prometheus/consoles",
                                    "--web.enable-lifecycle"
                                ],
                                volume_mounts=[
                                    client.V1VolumeMount(
                                        name="prometheus-config",
                                        mount_path="/etc/prometheus/"
                                    ),
                                    client.V1VolumeMount(
                                        name="prometheus-storage",
                                        mount_path="/prometheus/"
                                    )
                                ]
                            )
                        ],
                        volumes=[
                            client.V1Volume(
                                name="prometheus-config",
                                config_map=client.V1ConfigMapVolumeSource(
                                    name="prometheus-config"
                                )
                            ),
                            client.V1Volume(
                                name="prometheus-storage",
                                empty_dir=client.V1EmptyDirVolumeSource()
                            )
                        ]
                    )
                )
            )
        )
        
        try:
            self.apps_v1.create_namespaced_deployment(namespace=namespace, body=deployment)
        except ApiException as e:
            if e.status == 409:  # Already exists
                self.apps_v1.replace_namespaced_deployment(
                    name="prometheus", namespace=namespace, body=deployment
                )
        
        # Prometheus Service
        service = client.V1Service(
            metadata=client.V1ObjectMeta(name="prometheus-service", namespace=namespace),
            spec=client.V1ServiceSpec(
                selector={"app": "prometheus"},
                ports=[client.V1ServicePort(port=9090, target_port=9090)],
                type="ClusterIP"
            )
        )
        
        try:
            self.core_v1.create_namespaced_service(namespace=namespace, body=service)
        except ApiException as e:
            if e.status == 409:  # Already exists
                self.core_v1.replace_namespaced_service(
                    name="prometheus-service", namespace=namespace, body=service
                )
        
        result['deployed'] = True
        return result

    def _create_prometheus_rbac(self, namespace: str) -> None:
        """Create RBAC resources for Prometheus"""
        # ServiceAccount
        service_account = client.V1ServiceAccount(
            metadata=client.V1ObjectMeta(name="prometheus", namespace=namespace)
        )
        
        try:
            self.core_v1.create_namespaced_service_account(namespace=namespace, body=service_account)
        except ApiException as e:
            if e.status != 409:  # Ignore if already exists
                raise e
        
        # ClusterRole
        rbac_v1 = client.RbacAuthorizationV1Api()
        cluster_role = client.V1ClusterRole(
            metadata=client.V1ObjectMeta(name="prometheus"),
            rules=[
                client.V1PolicyRule(
                    api_groups=[""],
                    resources=["nodes", "nodes/proxy", "services", "endpoints", "pods"],
                    verbs=["get", "list", "watch"]
                ),
                client.V1PolicyRule(
                    api_groups=["extensions"],
                    resources=["ingresses"],
                    verbs=["get", "list", "watch"]
                )
            ]
        )
        
        # Add non-resource URL rule with version compatibility
        try:
            # Try the newer parameter name first
            non_resource_rule = client.V1PolicyRule(
                non_resource_ur_ls=["/metrics"],
                verbs=["get"]
            )
        except TypeError:
            # Fall back to older parameter name
            non_resource_rule = client.V1PolicyRule(
                non_resource_urls=["/metrics"],
                verbs=["get"]
            )
        
        cluster_role.rules.append(non_resource_rule)
        
        try:
            rbac_v1.create_cluster_role(body=cluster_role)
        except ApiException as e:
            if e.status != 409:  # Ignore if already exists
                raise e
        
        # ClusterRoleBinding
        # Create subject with version compatibility
        try:
            # Try V1Subject first (older versions)
            subject = client.V1Subject(
                kind="ServiceAccount",
                name="prometheus",
                namespace=namespace
            )
        except AttributeError:
            # Try RbacV1Subject (newer versions)
            try:
                subject = client.RbacV1Subject(
                    kind="ServiceAccount",
                    name="prometheus",
                    namespace=namespace
                )
            except AttributeError:
                # Manual construction as fallback
                subject = {
                    'kind': 'ServiceAccount',
                    'name': 'prometheus',
                    'namespace': namespace
                }
        
        cluster_role_binding = client.V1ClusterRoleBinding(
            metadata=client.V1ObjectMeta(name="prometheus"),
            subjects=[subject],
            role_ref=client.V1RoleRef(
                kind="ClusterRole",
                name="prometheus",
                api_group="rbac.authorization.k8s.io"
            )
        )
        
        try:
            rbac_v1.create_cluster_role_binding(body=cluster_role_binding)
        except ApiException as e:
            if e.status != 409:  # Ignore if already exists
                raise e

    def _deploy_grafana(self, namespace: str, service_type: str = "NodePort") -> Dict[str, Any]:
        """Deploy Grafana to the cluster"""
        result = {
            'deployed': False,
            'service_name': 'grafana-service',
            'service_type': service_type,
            'admin_user': 'admin',
            'admin_password': 'admin123'
        }
        
        # Grafana Deployment
        deployment = client.V1Deployment(
            metadata=client.V1ObjectMeta(name="grafana", namespace=namespace),
            spec=client.V1DeploymentSpec(
                replicas=1,
                selector=client.V1LabelSelector(match_labels={"app": "grafana"}),
                template=client.V1PodTemplateSpec(
                    metadata=client.V1ObjectMeta(labels={"app": "grafana"}),
                    spec=client.V1PodSpec(
                        containers=[
                            client.V1Container(
                                name="grafana",
                                image="grafana/grafana:latest",
                                ports=[client.V1ContainerPort(container_port=3000)],
                                env=[
                                    client.V1EnvVar(name="GF_SECURITY_ADMIN_USER", value="admin"),
                                    client.V1EnvVar(name="GF_SECURITY_ADMIN_PASSWORD", value="admin123"),
                                    client.V1EnvVar(name="GF_INSTALL_PLUGINS", value="grafana-kubernetes-app")
                                ],
                                volume_mounts=[
                                    client.V1VolumeMount(
                                        name="grafana-storage",
                                        mount_path="/var/lib/grafana"
                                    )
                                ]
                            )
                        ],
                        volumes=[
                            client.V1Volume(
                                name="grafana-storage",
                                empty_dir=client.V1EmptyDirVolumeSource()
                            )
                        ]
                    )
                )
            )
        )
        
        try:
            self.apps_v1.create_namespaced_deployment(namespace=namespace, body=deployment)
        except ApiException as e:
            if e.status == 409:  # Already exists
                self.apps_v1.replace_namespaced_deployment(
                    name="grafana", namespace=namespace, body=deployment
                )
        
        # Grafana Service
        service_ports = [client.V1ServicePort(port=3000, target_port=3000)]
        if service_type == "NodePort":
            service_ports[0].node_port = 30300
        
        service = client.V1Service(
            metadata=client.V1ObjectMeta(name="grafana-service", namespace=namespace),
            spec=client.V1ServiceSpec(
                selector={"app": "grafana"},
                ports=service_ports,
                type=service_type
            )
        )
        
        try:
            self.core_v1.create_namespaced_service(namespace=namespace, body=service)
            result['deployed'] = True
        except ApiException as e:
            if e.status == 409:  # Already exists
                self.core_v1.replace_namespaced_service(
                    name="grafana-service", namespace=namespace, body=service
                )
                result['deployed'] = True
        
        return result

    def _wait_for_monitoring_ready(self, namespace: str, timeout: int = 300) -> bool:
        """Wait for Prometheus and Grafana to be ready"""
        deployments = ["prometheus", "grafana"]
        start_time = time.time()
        
        for deployment_name in deployments:
            print(f"⏳ Waiting for {deployment_name} to be ready...")
            deployment_ready = False
            
            while time.time() - start_time < timeout and not deployment_ready:
                try:
                    deployment = self.apps_v1.read_namespaced_deployment(
                        name=deployment_name, namespace=namespace
                    )
                    
                    if (deployment.status.ready_replicas == deployment.spec.replicas and 
                        deployment.status.ready_replicas > 0):
                        print(f"✅ {deployment_name} is ready")
                        deployment_ready = True
                    else:
                        ready = deployment.status.ready_replicas or 0
                        total = deployment.spec.replicas
                        print(f"⏳ {deployment_name}: {ready}/{total} replicas ready...")
                        time.sleep(5)
                        
                except ApiException as e:
                    print(f"❌ Error checking {deployment_name} status: {e}")
                    return False
            
            if not deployment_ready:
                print(f"❌ Timeout waiting for {deployment_name} to be ready")
                return False
        
        return True

    def _configure_grafana(self, namespace: str, import_dashboard: bool = True) -> bool:
        """Configure Grafana with Prometheus data source and dashboard"""
        try:
            import requests
            import json
            
            # Get Grafana service URL
            grafana_url = self._get_grafana_url(namespace)
            if not grafana_url:
                print("❌ Could not determine Grafana URL")
                return False
            
            print(f"🔧 Configuring Grafana at {grafana_url}")
            
            # Wait for Grafana to be accessible with retry logic
            max_retries = 12  # 2 minutes total (12 * 10 seconds)
            retry_count = 0
            auth = ('admin', 'admin123')
            
            while retry_count < max_retries:
                try:
                    # Test connection to Grafana
                    health_response = requests.get(
                        f"{grafana_url}/api/health",
                        auth=auth,
                        timeout=10
                    )
                    if health_response.status_code == 200:
                        print("✅ Grafana is accessible")
                        break
                except Exception as e:
                    retry_count += 1
                    if retry_count < max_retries:
                        print(f"⏳ Waiting for Grafana to be accessible... (attempt {retry_count}/{max_retries})")
                        time.sleep(10)
                    else:
                        print(f"❌ Grafana not accessible after {max_retries} attempts")
                        print("💡 You can manually add Prometheus as a data source in Grafana")
                        return False
            
            # Add Prometheus data source
            datasource_payload = {
                "name": "Prometheus",
                "type": "prometheus",
                "url": f"http://prometheus-service.{namespace}.svc.cluster.local:9090",
                "access": "proxy",
                "isDefault": True
            }
            
            # Try to add data source with retry
            for attempt in range(3):
                try:
                    response = requests.post(
                        f"{grafana_url}/api/datasources",
                        json=datasource_payload,
                        auth=auth,
                        timeout=30
                    )
                    
                    if response.status_code in [200, 409]:  # Success or already exists
                        print("✅ Prometheus data source configured")
                        break
                    else:
                        print(f"⚠️  Attempt {attempt + 1}: Could not add Prometheus data source: {response.text}")
                        if attempt < 2:
                            time.sleep(5)
                except Exception as e:
                    print(f"⚠️  Attempt {attempt + 1}: Error adding data source: {e}")
                    if attempt < 2:
                        time.sleep(5)
            
            # Import default dashboard if requested
            if import_dashboard:
                self._import_kubernetes_dashboard(grafana_url, auth)
            
            return True
            
        except Exception as e:
            print(f"⚠️  Warning: Could not configure Grafana automatically: {e}")
            print("💡 You can manually add Prometheus as a data source in Grafana")
            return False

    def _get_grafana_url(self, namespace: str) -> Optional[str]:
        """Get Grafana service URL"""
        try:
            service = self.core_v1.read_namespaced_service(
                name="grafana-service", namespace=namespace
            )
            
            if service.spec.type == "NodePort":
                # Try to get node IP
                nodes = self.core_v1.list_node()
                if nodes.items:
                    # Get external IP or internal IP
                    node_ip = None
                    for node in nodes.items:
                        for address in node.status.addresses:
                            if address.type == "ExternalIP":
                                node_ip = address.address
                                break
                        if not node_ip:
                            for address in node.status.addresses:
                                if address.type == "InternalIP":
                                    node_ip = address.address
                                    break
                        if node_ip:
                            break
                    
                    if node_ip:
                        node_port = None
                        for port in service.spec.ports:
                            if port.node_port:
                                node_port = port.node_port
                                break
                        
                        if node_port:
                            return f"http://{node_ip}:{node_port}"
            
            elif service.spec.type == "LoadBalancer":
                if service.status.load_balancer.ingress:
                    ingress = service.status.load_balancer.ingress[0]
                    host = ingress.ip or ingress.hostname
                    if host:
                        return f"http://{host}:3000"
            
            # Fallback to port-forward approach
            return None
            
        except Exception as e:
            print(f"Warning: Could not determine Grafana URL: {e}")
            return None

    def _import_kubernetes_dashboard(self, grafana_url: str, auth: tuple) -> None:
        """Import a default Kubernetes dashboard"""
        try:
            import requests
            
            # Simple Kubernetes cluster dashboard JSON
            dashboard_json = {
                "dashboard": {
                    "id": None,
                    "title": "Kubernetes Cluster Overview",
                    "tags": ["kubernetes"],
                    "timezone": "browser",
                    "panels": [
                        {
                            "id": 1,
                            "title": "Cluster CPU Usage",
                            "type": "stat",
                            "targets": [
                                {
                                    "expr": "100 - (avg(irate(node_cpu_seconds_total{mode=\"idle\"}[5m])) * 100)",
                                    "refId": "A"
                                }
                            ],
                            "gridPos": {"h": 8, "w": 12, "x": 0, "y": 0}
                        },
                        {
                            "id": 2,
                            "title": "Cluster Memory Usage",
                            "type": "stat",
                            "targets": [
                                {
                                    "expr": "100 * (1 - ((avg_over_time(node_memory_MemFree_bytes[10m]) + avg_over_time(node_memory_Cached_bytes[10m]) + avg_over_time(node_memory_Buffers_bytes[10m])) / avg_over_time(node_memory_MemTotal_bytes[10m])))",
                                    "refId": "A"
                                }
                            ],
                            "gridPos": {"h": 8, "w": 12, "x": 12, "y": 0}
                        },
                        {
                            "id": 3,
                            "title": "Pod Count",
                            "type": "stat",
                            "targets": [
                                {
                                    "expr": "sum(kube_pod_info)",
                                    "refId": "A"
                                }
                            ],
                            "gridPos": {"h": 8, "w": 12, "x": 0, "y": 8}
                        }
                    ],
                    "time": {"from": "now-1h", "to": "now"},
                    "refresh": "30s"
                },
                "overwrite": True
            }
            
            response = requests.post(
                f"{grafana_url}/api/dashboards/db",
                json=dashboard_json,
                auth=auth,
                timeout=30
            )
            
            if response.status_code == 200:
                print("✅ Kubernetes dashboard imported")
            else:
                print(f"⚠️  Warning: Could not import dashboard: {response.text}")
                
        except Exception as e:
            print(f"⚠️  Warning: Could not import dashboard: {e}")

    def get_monitoring_info(self, namespace: str = "monitoring") -> Dict[str, Any]:
        """Get monitoring stack information and URLs"""
        result = {
            'namespace': namespace,
            'prometheus': {'running': False, 'url': None},
            'grafana': {'running': False, 'url': None, 'credentials': None},
            'services': []
        }
        
        try:
            # Check if deployments are running
            deployments = self.apps_v1.list_namespaced_deployment(namespace=namespace)
            
            for deployment in deployments.items:
                if deployment.metadata.name == "prometheus":
                    result['prometheus']['running'] = (
                        deployment.status.ready_replicas == deployment.spec.replicas and
                        deployment.status.ready_replicas > 0
                    )
                elif deployment.metadata.name == "grafana":
                    result['grafana']['running'] = (
                        deployment.status.ready_replicas == deployment.spec.replicas and
                        deployment.status.ready_replicas > 0
                    )
            
            # Get service information
            services = self.core_v1.list_namespaced_service(namespace=namespace)
            
            for service in services.items:
                service_info = {
                    'name': service.metadata.name,
                    'type': service.spec.type,
                    'cluster_ip': service.spec.cluster_ip,
                    'ports': []
                }
                
                for port in service.spec.ports:
                    port_info = {
                        'port': port.port,
                        'target_port': port.target_port,
                        'protocol': port.protocol
                    }
                    if hasattr(port, 'node_port') and port.node_port:
                        port_info['node_port'] = port.node_port
                    service_info['ports'].append(port_info)
                
                result['services'].append(service_info)
                
                # Set URLs
                if service.metadata.name == "grafana-service":
                    result['grafana']['credentials'] = {
                        'username': 'admin',
                        'password': 'admin123'
                    }
                    
                    if service.spec.type == "NodePort":
                        # Get node IP and port
                        url_info = self.get_service_url("grafana-service", namespace)
                        if url_info and url_info.get('external_url'):
                            result['grafana']['url'] = url_info['external_url']
                
                elif service.metadata.name == "prometheus-service":
                    if service.spec.type == "NodePort":
                        url_info = self.get_service_url("prometheus-service", namespace)
                        if url_info and url_info.get('external_url'):
                            result['prometheus']['url'] = url_info['external_url']
            
            return result
            
        except ApiException as e:
            result['error'] = str(e)
            return result

    def add_prometheus_target(self, job_name: str, targets: List[str], 
                            namespace: str = "monitoring", 
                            metrics_path: str = "/metrics",
                            scrape_interval: str = "15s") -> bool:
        """
        Add a new scrape target to Prometheus configuration
        
        Args:
            job_name: Name for the job in Prometheus config
            targets: List of target addresses (e.g., ["service:port", "1.2.3.4:9090"])
            namespace: Monitoring namespace
            metrics_path: Path to metrics endpoint
            scrape_interval: How often to scrape the target
        
        Returns:
            bool: True if target was added successfully
        """
        try:
            # Get current ConfigMap
            configmap = self.core_v1.read_namespaced_config_map(
                name="prometheus-config", 
                namespace=namespace
            )
            
            current_config = configmap.data.get("prometheus.yml", "")
            
            # Parse YAML to add new target
            import yaml
            config_data = yaml.safe_load(current_config)
            
            # Check if job already exists
            existing_jobs = [job['job_name'] for job in config_data.get('scrape_configs', [])]
            if job_name in existing_jobs:
                print(f"❌ Job '{job_name}' already exists. Use update or remove it first.")
                return False
            
            # Create new job config
            new_job = {
                'job_name': job_name,
                'scrape_interval': scrape_interval,
                'metrics_path': metrics_path,
                'static_configs': [
                    {'targets': targets}
                ]
            }
            
            # Add to scrape configs
            if 'scrape_configs' not in config_data:
                config_data['scrape_configs'] = []
            
            config_data['scrape_configs'].append(new_job)
            
            # Convert back to YAML
            updated_config = yaml.dump(config_data, default_flow_style=False, sort_keys=False)
            
            # Update ConfigMap
            configmap.data["prometheus.yml"] = updated_config
            self.core_v1.replace_namespaced_config_map(
                name="prometheus-config",
                namespace=namespace,
                body=configmap
            )
            
            # Restart Prometheus deployment to pick up new config
            self._restart_prometheus_deployment(namespace)
            
            print(f"✅ Added Prometheus target job '{job_name}' with targets: {targets}")
            return True
            
        except Exception as e:
            print(f"❌ Error adding Prometheus target: {e}")
            return False

    def remove_prometheus_target(self, job_name: str, namespace: str = "monitoring") -> bool:
        """
        Remove a scrape target from Prometheus configuration
        
        Args:
            job_name: Name of the job to remove
            namespace: Monitoring namespace
        
        Returns:
            bool: True if target was removed successfully
        """
        try:
            # Get current ConfigMap
            configmap = self.core_v1.read_namespaced_config_map(
                name="prometheus-config", 
                namespace=namespace
            )
            
            current_config = configmap.data.get("prometheus.yml", "")
            
            # Parse YAML to remove target
            import yaml
            config_data = yaml.safe_load(current_config)
            
            # Find and remove the job
            original_jobs = config_data.get('scrape_configs', [])
            updated_jobs = [job for job in original_jobs if job.get('job_name') != job_name]
            
            if len(original_jobs) == len(updated_jobs):
                print(f"❌ Job '{job_name}' not found in Prometheus configuration")
                return False
            
            config_data['scrape_configs'] = updated_jobs
            
            # Convert back to YAML
            updated_config = yaml.dump(config_data, default_flow_style=False, sort_keys=False)
            
            # Update ConfigMap
            configmap.data["prometheus.yml"] = updated_config
            self.core_v1.replace_namespaced_config_map(
                name="prometheus-config",
                namespace=namespace,
                body=configmap
            )
            
            # Restart Prometheus deployment to pick up new config
            self._restart_prometheus_deployment(namespace)
            
            print(f"✅ Removed Prometheus target job '{job_name}'")
            return True
            
        except Exception as e:
            print(f"❌ Error removing Prometheus target: {e}")
            return False

    def list_prometheus_targets(self, namespace: str = "monitoring") -> Dict[str, Any]:
        """
        List all configured Prometheus targets
        
        Args:
            namespace: Monitoring namespace
        
        Returns:
            Dict containing target information
        """
        try:
            # Get current ConfigMap
            configmap = self.core_v1.read_namespaced_config_map(
                name="prometheus-config", 
                namespace=namespace
            )
            
            current_config = configmap.data.get("prometheus.yml", "")
            
            # Parse YAML to get targets
            import yaml
            config_data = yaml.safe_load(current_config)
            
            targets = []
            for job in config_data.get('scrape_configs', []):
                job_info = {
                    'job_name': job.get('job_name', 'unknown'),
                    'scrape_interval': job.get('scrape_interval', 'default'),
                    'metrics_path': job.get('metrics_path', '/metrics'),
                    'targets': []
                }
                
                # Get static targets
                if 'static_configs' in job:
                    for static_config in job['static_configs']:
                        job_info['targets'].extend(static_config.get('targets', []))
                
                # Mark kubernetes discovery jobs
                if 'kubernetes_sd_configs' in job:
                    job_info['type'] = 'kubernetes_discovery'
                    job_info['targets'] = ['<kubernetes_discovery>']
                else:
                    job_info['type'] = 'static'
                
                targets.append(job_info)
            
            return {
                'namespace': namespace,
                'targets': targets,
                'total_jobs': len(targets)
            }
            
        except Exception as e:
            return {'error': str(e)}

    def update_prometheus_target(self, job_name: str, targets: List[str],
                               namespace: str = "monitoring",
                               metrics_path: str = None,
                               scrape_interval: str = None) -> bool:
        """
        Update an existing Prometheus target
        
        Args:
            job_name: Name of the job to update
            targets: New list of target addresses
            namespace: Monitoring namespace
            metrics_path: Optional new metrics path
            scrape_interval: Optional new scrape interval
        
        Returns:
            bool: True if target was updated successfully
        """
        try:
            # Get current ConfigMap
            configmap = self.core_v1.read_namespaced_config_map(
                name="prometheus-config", 
                namespace=namespace
            )
            
            current_config = configmap.data.get("prometheus.yml", "")
            
            # Parse YAML to update target
            import yaml
            config_data = yaml.safe_load(current_config)
            
            # Find and update the job
            job_found = False
            for job in config_data.get('scrape_configs', []):
                if job.get('job_name') == job_name:
                    job_found = True
                    
                    # Update targets
                    if 'static_configs' not in job:
                        job['static_configs'] = [{}]
                    
                    job['static_configs'][0]['targets'] = targets
                    
                    # Update optional fields
                    if metrics_path:
                        job['metrics_path'] = metrics_path
                    if scrape_interval:
                        job['scrape_interval'] = scrape_interval
                    
                    break
            
            if not job_found:
                print(f"❌ Job '{job_name}' not found in Prometheus configuration")
                return False
            
            # Convert back to YAML
            updated_config = yaml.dump(config_data, default_flow_style=False, sort_keys=False)
            
            # Update ConfigMap
            configmap.data["prometheus.yml"] = updated_config
            self.core_v1.replace_namespaced_config_map(
                name="prometheus-config",
                namespace=namespace,
                body=configmap
            )
            
            # Restart Prometheus deployment to pick up new config
            self._restart_prometheus_deployment(namespace)
            
            print(f"✅ Updated Prometheus target job '{job_name}' with targets: {targets}")
            return True
            
        except Exception as e:
            print(f"❌ Error updating Prometheus target: {e}")
            return False

    def _restart_prometheus_deployment(self, namespace: str) -> bool:
        """Restart Prometheus deployment to reload configuration"""
        try:
            # Get current deployment
            deployment = self.apps_v1.read_namespaced_deployment(
                name="prometheus", 
                namespace=namespace
            )
            
            # Update deployment to trigger restart
            if deployment.spec.template.metadata.annotations is None:
                deployment.spec.template.metadata.annotations = {}
            
            import time
            deployment.spec.template.metadata.annotations['kubectl.kubernetes.io/restartedAt'] = str(int(time.time()))
            
            # Apply the update
            self.apps_v1.patch_namespaced_deployment(
                name="prometheus",
                namespace=namespace,
                body=deployment
            )
            
            print("🔄 Restarting Prometheus deployment to reload configuration...")
            return True
            
        except Exception as e:
            print(f"⚠️  Could not restart Prometheus deployment: {e}")
            return False

    # ======================
    # HELM-BASED MONITORING METHODS
    # ======================
    
    def setup_helm_monitoring(self, namespace: str = "monitoring", 
                            grafana_service_type: str = "NodePort",
                            prometheus_storage_size: str = "10Gi",
                            grafana_storage_size: str = "5Gi",
                            wait_for_ready: bool = True,
                            install_ingress: bool = False) -> Dict:
        """Deploy monitoring stack using official Helm charts"""
        import subprocess
        import tempfile
        import os
        
        try:
            # Check if Helm is available, and install if not
            if not self._check_helm_available():
                print("🔧 Helm not found. Attempting automatic installation...")
                install_result = self._install_helm_automatically()
                
                if not install_result['success']:
                    return {
                        'success': False,
                        'error': f"Helm is required but could not be installed automatically. {install_result.get('error', '')}",
                        'suggestion': 'Please install Helm manually: https://helm.sh/docs/intro/install/'
                    }
                else:
                    print(f"✅ {install_result['message']}")
                    
                    # Verify installation worked
                    if not self._check_helm_available():
                        return {
                            'success': False,
                            'error': 'Helm was installed but is not accessible. You may need to restart your terminal or add Helm to your PATH.',
                            'suggestion': 'Please restart your terminal and try again, or install Helm manually.'
                        }
            
            print("✅ Helm is available")
            
            # Create namespace if it doesn't exist
            try:
                self.core_v1.create_namespace(
                    body=client.V1Namespace(metadata=client.V1ObjectMeta(name=namespace))
                )
                print(f"✅ Created namespace: {namespace}")
            except ApiException as e:
                if e.status == 409:  # Already exists
                    print(f"✅ Namespace {namespace} already exists")
                else:
                    print(f"⚠️  Could not create namespace: {e}")
            
            # Add Prometheus community Helm repository
            print("📦 Adding Prometheus community Helm repository...")
            try:
                subprocess.run([
                    'helm', 'repo', 'add', 'prometheus-community', 
                    'https://prometheus-community.github.io/helm-charts'
                ], check=True, capture_output=True, text=True)
                
                subprocess.run(['helm', 'repo', 'update'], check=True, capture_output=True, text=True)
                print("✅ Helm repository added and updated")
            except subprocess.CalledProcessError as e:
                return {
                    'success': False,
                    'error': f'Failed to add Helm repository: {e.stderr if e.stderr else str(e)}'
                }
            
            # Create Helm values file
            helm_values = {
                'grafana': {
                    'enabled': True,
                    'persistence': {
                        'enabled': True,
                        'size': grafana_storage_size
                    },
                    'service': {
                        'type': grafana_service_type
                    },
                    'adminPassword': 'admin',
                    'datasources': {
                        'datasources.yaml': {
                            'apiVersion': 1,
                            'datasources': [{
                                'name': 'Prometheus',
                                'type': 'prometheus',
                                'url': 'http://kube-prometheus-stack-prometheus:9090',
                                'access': 'proxy',
                                'isDefault': True
                            }]
                        }
                    },
                    'dashboardProviders': {
                        'dashboardproviders.yaml': {
                            'apiVersion': 1,
                            'providers': [{
                                'name': 'default',
                                'orgId': 1,
                                'folder': '',
                                'type': 'file',
                                'disableDeletion': False,
                                'editable': True,
                                'options': {
                                    'path': '/var/lib/grafana/dashboards/default'
                                }
                            }]
                        }
                    },
                    'dashboards': {
                        'default': {
                            'kubernetes-cluster-dashboard': {
                                'gnetId': 7249,
                                'revision': 1,
                                'datasource': 'Prometheus'
                            },
                            'kubernetes-pod-dashboard': {
                                'gnetId': 6417,
                                'revision': 1,
                                'datasource': 'Prometheus'
                            },
                            'node-exporter-dashboard': {
                                'gnetId': 1860,
                                'revision': 27,
                                'datasource': 'Prometheus'
                            }
                        }
                    }
                },
                'prometheus': {
                    'enabled': True,
                    'prometheusSpec': {
                        'retention': '30d',
                        'storageSpec': {
                            'volumeClaimTemplate': {
                                'spec': {
                                    'accessModes': ['ReadWriteOnce'],
                                    'resources': {
                                        'requests': {
                                            'storage': prometheus_storage_size
                                        }
                                    }
                                }
                            }
                        },
                        'serviceMonitorSelectorNilUsesHelmValues': False
                    }
                },
                'alertmanager': {
                    'enabled': True
                },
                'nodeExporter': {
                    'enabled': True
                },
                'kubeStateMetrics': {
                    'enabled': True
                },
                'defaultRules': {
                    'create': True,
                    'rules': {
                        'alertmanager': True,
                        'etcd': True,
                        'general': True,
                        'k8s': True,
                        'kubeApiserver': True,
                        'kubePrometheusNodeRecording': True,
                        'kubernetesApps': True,
                        'kubernetesResources': True,
                        'kubernetesStorage': True,
                        'kubernetesSystem': True,
                        'network': True,
                        'node': True,
                        'prometheus': True,
                        'prometheusOperator': True
                    }
                }
            }
            
            # Add ingress if requested
            if install_ingress:
                helm_values['grafana']['ingress'] = {
                    'enabled': True,
                    'hosts': [f'grafana.{namespace}.local'],
                    'paths': ['/']
                }
            
            # Write values to temporary file
            with tempfile.NamedTemporaryFile(mode='w', suffix='.yaml', delete=False) as f:
                yaml.dump(helm_values, f, default_flow_style=False)
                values_file = f.name
            
            try:
                # Install the Helm chart
                print("🚀 Installing kube-prometheus-stack via Helm...")
                helm_cmd = [
                    'helm', 'install', 'kube-prometheus-stack',
                    'prometheus-community/kube-prometheus-stack',
                    '--namespace', namespace,
                    '--values', values_file
                ]
                
                if wait_for_ready:
                    helm_cmd.append('--wait')
                    helm_cmd.extend(['--timeout', '10m'])
                
                result = subprocess.run(helm_cmd, capture_output=True, text=True, check=True)
                print("✅ Helm chart installed successfully")
                
                # Wait a bit for pods to start
                if wait_for_ready:
                    print("⏳ Waiting for pods to be ready...")
                    time.sleep(30)
                
                # Get service information
                services_info = self._get_helm_monitoring_services(namespace)
                
                return {
                    'success': True,
                    'namespace': namespace,
                    'release_name': 'kube-prometheus-stack',
                    'prometheus': {'deployed': True},
                    'grafana': {
                        'deployed': True,
                        'admin_password': 'admin'
                    },
                    'grafana_url': services_info.get('grafana_url'),
                    'prometheus_url': services_info.get('prometheus_url'),
                    'alertmanager_url': services_info.get('alertmanager_url')
                }
                
            finally:
                # Clean up temporary file
                os.unlink(values_file)
                
        except subprocess.CalledProcessError as e:
            error_msg = e.stderr if e.stderr else str(e)
            return {
                'success': False,
                'error': f'Helm installation failed: {error_msg}'
            }
        except Exception as e:
            return {
                'success': False,
                'error': f'Failed to setup Helm monitoring: {str(e)}'
            }
    
    def get_helm_monitoring_info(self, namespace: str = "monitoring") -> Dict:
        """Get information about the Helm-based monitoring stack"""
        import subprocess
        
        try:
            # Check if Helm release exists
            try:
                result = subprocess.run([
                    'helm', 'status', 'kube-prometheus-stack', 
                    '--namespace', namespace
                ], capture_output=True, text=True, check=True)
                
                # Parse Helm status
                lines = result.stdout.split('\n')
                release_info = {}
                for line in lines:
                    if 'STATUS:' in line:
                        release_info['release_status'] = line.split('STATUS:')[1].strip()
                    elif 'LAST DEPLOYED:' in line:
                        release_info['last_deployed'] = line.split('LAST DEPLOYED:')[1].strip()
                
            except subprocess.CalledProcessError:
                return {'error': 'Helm release not found. Use setup-helm-monitoring to deploy first.'}
            
            # Get services information
            services_info = self._get_helm_monitoring_services(namespace)
            
            # Get pod status
            pods_info = self._get_monitoring_pods_status(namespace)
            
            return {
                'release_name': 'kube-prometheus-stack',
                'release_status': release_info.get('release_status', 'Unknown'),
                'last_deployed': release_info.get('last_deployed', 'Unknown'),
                'prometheus': {
                    'running': any(pod['name'].startswith('prometheus-kube-prometheus-stack-prometheus') 
                                 for pod in pods_info if pod['status'] == 'Running'),
                    'url': services_info.get('prometheus_url', 'Port-forward required')
                },
                'grafana': {
                    'running': any(pod['name'].startswith('kube-prometheus-stack-grafana') 
                                 for pod in pods_info if pod['status'] == 'Running'),
                    'url': services_info.get('grafana_url', 'Port-forward required')
                },
                'alertmanager': {
                    'running': any(pod['name'].startswith('alertmanager-kube-prometheus-stack-alertmanager') 
                                 for pod in pods_info if pod['status'] == 'Running'),
                    'url': services_info.get('alertmanager_url', 'Port-forward required')
                },
                'pods': pods_info
            }
            
        except Exception as e:
            return {'error': f'Failed to get monitoring info: {str(e)}'}
    
    def delete_helm_monitoring(self, namespace: str = "monitoring", 
                             release_name: str = "kube-prometheus-stack") -> Dict:
        """Delete Helm-based monitoring stack"""
        import subprocess
        
        try:
            # Uninstall Helm release
            result = subprocess.run([
                'helm', 'uninstall', release_name,
                '--namespace', namespace
            ], capture_output=True, text=True, check=True)
            
            print(f"✅ Helm release '{release_name}' uninstalled")
            
            # Count remaining resources (optional cleanup)
            try:
                # Delete PVCs that might remain
                pvcs = self.core_v1.list_namespaced_persistent_volume_claim(namespace=namespace)
                pvc_count = 0
                for pvc in pvcs.items:
                    if 'prometheus' in pvc.metadata.name or 'grafana' in pvc.metadata.name:
                        self.core_v1.delete_namespaced_persistent_volume_claim(
                            name=pvc.metadata.name, 
                            namespace=namespace
                        )
                        pvc_count += 1
                
                if pvc_count > 0:
                    print(f"✅ Cleaned up {pvc_count} persistent volume claims")
                
            except Exception as cleanup_error:
                print(f"⚠️  Could not clean up some resources: {cleanup_error}")
            
            return {
                'success': True,
                'resources_deleted': pvc_count
            }
            
        except subprocess.CalledProcessError as e:
            error_msg = e.stderr if e.stderr else str(e)
            return {
                'success': False,
                'error': f'Failed to uninstall Helm release: {error_msg}'
            }
        except Exception as e:
            return {
                'success': False,
                'error': f'Failed to delete monitoring stack: {str(e)}'
            }
    
    def _get_helm_monitoring_services(self, namespace: str) -> Dict:
        """Get service URLs for Helm monitoring components"""
        services_info = {}
        
        try:
            # Get services
            services = self.core_v1.list_namespaced_service(namespace=namespace)
            
            for service in services.items:
                service_name = service.metadata.name
                
                if 'grafana' in service_name:
                    url = self._get_service_url(service, namespace, 80)
                    if url:
                        services_info['grafana_url'] = url
                
                elif 'prometheus' in service_name and 'operated' not in service_name:
                    url = self._get_service_url(service, namespace, 9090)
                    if url:
                        services_info['prometheus_url'] = url
                
                elif 'alertmanager' in service_name and 'operated' not in service_name:
                    url = self._get_service_url(service, namespace, 9093)
                    if url:
                        services_info['alertmanager_url'] = url
        
        except Exception as e:
            print(f"⚠️  Could not get service information: {e}")
        
        return services_info
    
    def _get_monitoring_pods_status(self, namespace: str) -> List[Dict]:
        """Get status of monitoring pods"""
        pods_info = []
        
        try:
            pods = self.core_v1.list_namespaced_pod(namespace=namespace)
            
            for pod in pods.items:
                if any(component in pod.metadata.name for component in 
                      ['prometheus', 'grafana', 'alertmanager', 'node-exporter', 'kube-state-metrics']):
                    
                    ready_containers = 0
                    total_containers = len(pod.status.container_statuses) if pod.status.container_statuses else 0
                    
                    if pod.status.container_statuses:
                        ready_containers = sum(1 for cs in pod.status.container_statuses if cs.ready)
                    
                    pods_info.append({
                        'name': pod.metadata.name,
                        'status': pod.status.phase,
                        'ready': ready_containers,
                        'total': total_containers
                    })
        
        except Exception as e:
            print(f"⚠️  Could not get pod status: {e}")
        
        return pods_info
