#!/usr/bin/env python3
"""
Download KEGG organism information
"""
import sys
import logging
import re
from pathlib import Path
from typing import List, Dict
import urllib.request
from html.parser import HTMLParser

from ..utils import polars_utils, parsers
from ..config import config

LOG = logging.getLogger(__name__)


class KEGGOrgHTMLParser(HTMLParser):
    """Parser for KEGG organism HTML table with rowspan handling."""
    
    def __init__(self):
        super().__init__()
        self.organisms = []
        self.current_row = {}
        self.cell_index = 0
        self.in_organism_row = False
        self.in_organism_name = False
        self.expecting_org_code = False
        self.rowspan_counter = 0  # Track how many category cells to skip
        
    def handle_starttag(self, tag, attrs):
        attrs_dict = dict(attrs)
        
        if tag == 'tr':
            # Start of a new row
            self.current_row = {
                'org_code': '',
                'name': '',
                'url': ''
            }
            self.cell_index = 0
            self.in_organism_row = False
            self.in_organism_name = False
            self.expecting_org_code = False
            
            # Reset rowspan counter for new row
            self.rowspan_counter = 0
        
        elif tag == 'td':
            # This is a table cell
            self.cell_index += 1
            
            # Check for rowspan attribute (category cells)
            if 'rowspan' in attrs_dict:
                # This is a category cell, skip it
                self.rowspan_counter += 1
                LOG.debug(f"Skipping category cell with rowspan (total skipped: {self.rowspan_counter})")
                return
            
            # Calculate adjusted cell index (skip category cells)
            adjusted_index = self.cell_index - self.rowspan_counter
            
            # DEBUG: Log cell structure
            if LOG.level <= logging.DEBUG:
                LOG.debug(f"Cell {self.cell_index} (adjusted: {adjusted_index}): tag={tag}, attrs={attrs}")
            
            # Adjusted cell 1: Should be org_code link
            if adjusted_index == 1:
                self.expecting_org_code = True
                self.in_organism_row = True
            
            # Adjusted cell 3: Should be organism name (plain text, no links)
            elif adjusted_index == 3 and self.in_organism_row:
                self.in_organism_name = True
            
            # Adjusted cell 5: Should be FTP URL link (last column)
            elif adjusted_index == 5 and self.in_organism_row:
                # This cell should contain the FTP URL
                pass
        
        elif tag == 'a' and self.in_organism_row:
            href = attrs_dict.get('href', '')
            
            # Calculate adjusted cell index
            adjusted_index = self.cell_index - self.rowspan_counter
            
            # Adjusted cell 1: Extract org_code from link
            if adjusted_index == 1 and self.expecting_org_code:
                match = re.search(r'org=([a-zA-Z0-9]+)', href)
                if match:
                    self.current_row['org_code'] = match.group(1)
                    LOG.debug(f"Found org_code: {self.current_row['org_code']}")
                    self.expecting_org_code = False
            
            # Adjusted cell 5: Extract FTP URL
            elif adjusted_index == 5:
                if href and ('ftp.ncbi.nlm.nih.gov' in href or 'http' in href or 'https://' in href):
                    self.current_row['url'] = href
                    LOG.debug(f"Found FTP URL: {href[:80]}...")
    
    def handle_data(self, data):
        # Calculate adjusted cell index
        adjusted_index = self.cell_index - self.rowspan_counter
        
        # Collect organism name from adjusted cell 3 (plain text)
        if self.in_organism_name and data.strip() and adjusted_index == 3:
            # Append to name (might be split across multiple text nodes)
            if self.current_row['name']:
                self.current_row['name'] += ' ' + data.strip()
            else:
                self.current_row['name'] = data.strip()
            LOG.debug(f"Found organism name part: '{data.strip()}'")
    
    def handle_endtag(self, tag):
        if tag == 'td' and self.in_organism_name:
            # End of organism name cell
            self.in_organism_name = False
        
        elif tag == 'tr' and self.in_organism_row:
            # End of an organism row
            if self.current_row['org_code'] and self.current_row['name']:
                # Clean up the name
                self.current_row['name'] = re.sub(r'\s+', ' ', self.current_row['name']).strip()
                
                # Validate we have data
                LOG.debug(f"Completed organism: {self.current_row['org_code']} = '{self.current_row['name']}'")
                
                self.organisms.append(self.current_row.copy())
            elif self.current_row['org_code']:
                LOG.warning(f"Incomplete organism row for {self.current_row['org_code']}")
            
            self.in_organism_row = False
            self.cell_index = 0

# OLD MODEL parse_kegg_html_simple_regex()
#
# Due to error(?) at KEGG https://www.kegg.jp/kegg/tables/br08606.html
#   hsa (human) and mmu (house mouse) will link to this
#   hsa: https://ftp.ncbi.nlm.nih.gov/refseq/H_sapiens/mRNA_Prot/
#   mmu: https://ftp.ncbi.nlm.nih.gov/refseq/release/complete/
# Instead of:
#   hsa: https://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/000/001/405/GCF_000001405.40_GRCh38.p14/
#   mmu: https://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/000/001/635/GCF_000001635.27_GRCm39/
# Still we retain this function, in case we are wrong!
#
# def parse_kegg_html_simple_regex(html_content: str) -> List[Dict]:
    # """
    # Robust regex parser for KEGG organism table.
    # Handles rowspan attributes and complex table structure.
    # """
    # LOG.info("Using robust regex parser for KEGG organism table")
    
    # organisms = []
    
    # # Extract just the table to work with a smaller string
    # table_pattern = r'<table[^>]*>.*?<tbody>(.*?)</tbody>.*?</table>'
    # table_match = re.search(table_pattern, html_content, re.DOTALL | re.IGNORECASE)
    
    # if not table_match:
        # LOG.warning("Could not find main table in HTML")
        # # Try to find any table
        # table_match = re.search(r'<tbody>(.*?)</tbody>', html_content, re.DOTALL | re.IGNORECASE)
        # if not table_match:
            # return organisms
    
    # table_html = table_match.group(1)
    
    # # Split into rows
    # rows = re.split(r'</tr>\s*<tr[^>]*>', table_html)
    
    # LOG.info(f"Found {len(rows)} table rows")
    
    # current_category = ""
    # row_counter = 0
    
    # for row in rows:
        # row_counter += 1
        
        # # Skip empty rows
        # if not row.strip():
            # continue
        
        # # Check if this row contains an organism code
        # org_code_match = re.search(r'org=([a-zA-Z0-9]{3,4})', row, re.IGNORECASE)
        
        # if org_code_match:
            # org_code = org_code_match.group(1)
            
            # # Extract organism name - look for text in a <td> that's not a link
            # # The name is typically after the KGB link cell
            # name_pattern = r'''
                # org=[^>]+>  # After org link
                # (?:[^<]*</a>\s*</td>\s*<td[^>]*>\s*(?:<a[^>]*>[^<]*</a>\s*)?</td>\s*)?  # Skip KGB cell
                # <td[^>]*>\s*
                # ([^<]+?)\s*  # organism name
                # </td>
            # '''
            
            # name_match = re.search(name_pattern, row, re.VERBOSE | re.IGNORECASE | re.DOTALL)
            
            # if not name_match:
                # # Try alternative pattern
                # name_match = re.search(r'<td[^>]*>\s*([^<]+?\([^)]+\))\s*</td>', row, re.IGNORECASE)
            
            # org_name = name_match.group(1).strip() if name_match else ""
            
            # # Extract FTP URL - look for ftp.ncbi.nlm.nih.gov in links
            # url_pattern = r'<a\s+href="([^"]*ftp\.ncbi\.nlm\.nih\.gov[^"]*)"[^>]*>'
            # url_matches = re.findall(url_pattern, row, re.IGNORECASE)
            
            # ftp_url = ""
            # if url_matches:
                # # Take the last FTP URL in the row (should be the correct one)
                # ftp_url = url_matches[-1]
            # else:
                # # Try other URL patterns
                # url_pattern2 = r'<a\s+href="(https?://[^"]*)"[^>]*>RefSeq</a>'
                # url_match2 = re.search(url_pattern2, row, re.IGNORECASE)
                # if url_match2:
                    # ftp_url = url_match2.group(1)
            
            # # Clean up the name
            # org_name = re.sub(r'\s+', ' ', org_name).strip()
            
            # organisms.append({
                # 'org_code': org_code,
                # 'name': org_name,
                # 'url': ftp_url
            # })
            
            # # Debug output for first few
            # if len(organisms) <= 10:
                # url_preview = ftp_url[:60] + "..." if ftp_url and len(ftp_url) > 60 else ftp_url
                # LOG.debug(f"Row {row_counter}: {org_code} = '{org_name}' -> {url_preview or 'NO URL'}")
    
    # # Verify we have correct mappings
    # if organisms:
        # LOG.info("Verifying organism-URL mappings:")
        
        # # Check for known organisms
        # expected_mappings = {
            # 'hsa': ['H_sapiens', 'human'],
            # 'ptr': ['PanTro', 'chimpanzee'],
            # 'pps': ['PanPan', 'bonobo'],
            # 'mmu': ['musculus', 'mouse']
        # }
        
        # for org in organisms[:15]:
            # if org['url']:
                # org_code = org['org_code']
                # url_lower = org['url'].lower()
                # name_lower = org['name'].lower()
                
                # if org_code in expected_mappings:
                    # expected_terms = expected_mappings[org_code]
                    # found_any = any(term.lower() in url_lower or term.lower() in name_lower for term in expected_terms)
                    
                    # if not found_any:
                        # LOG.warning(f"  ⚠️  {org_code} URL may not match organism: {org['url'][:80]}...")
            # else:
                # LOG.warning(f"  ⚠️  {org['org_code']} has no FTP URL")
    
    # return organisms

def parse_kegg_html_simple_regex(html_content: str) -> List[Dict]:
    """
    Robust regex parser for KEGG organism table.
    Handles rowspan attributes and complex table structure.
    """
    LOG.info("Using robust regex parser for KEGG organism table")
    
    organisms = []
    
    # Extract all table rows first
    rows = re.findall(r'<tr[^>]*>(.*?)</tr>', html_content, re.DOTALL | re.IGNORECASE)
    LOG.info(f"Found {len(rows)} total table rows")
    
    if not rows:
        LOG.warning("No table rows found")
        return organisms
    
    # Skip the header row (first <tr> after the table start)
    rows = rows[1:]  # Skip header
    
    # Known genome URL mappings
    known_genome_urls = {
        'hsa': 'https://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/000/001/405/GCF_000001405.40_GRCh38.p14',
        'mmu': 'https://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/000/001/635/GCF_000001635.27_GRCm39',
        'rno': 'https://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/036/323/735/GCF_036323735.1_GRCr8',
    }
    
    row_counter = 0
    organism_count = 0
    
    for row in rows:
        row_counter += 1
        
        # Skip empty rows
        if not row.strip():
            continue
        
        # Debug: Log first few rows to see structure
        if row_counter <= 3:
            LOG.debug(f"Row {row_counter} (first 200 chars): {row[:200]}")
        
        # Look for organism code in the row
        org_code_match = re.search(r'org=([a-zA-Z0-9]{2,4})["\']', row, re.IGNORECASE)
        
        if org_code_match:
            org_code = org_code_match.group(1)
            organism_count += 1
            
            # Extract organism name - simpler approach
            # Look for the text in the third data cell after the org link
            # Pattern: org=...> </a></td><td ...></td><td ...>ORGANISM NAME HERE</td>
            
            # First try: Look for the pattern with KGB link cell
            name_match = re.search(
                r'org=[^>]+>.*?</a>.*?</td>.*?</td>.*?<td[^>]*>(.*?)</td>',
                row, 
                re.DOTALL | re.IGNORECASE
            )
            
            if not name_match:
                # Second try: Look for any td with organism-like name (contains parentheses)
                name_match = re.search(r'<td[^>]*>([^<]*\([^)]+\)[^<]*)</td>', row, re.IGNORECASE)
            
            if not name_match:
                # Third try: Just get the first non-empty td content after the org link
                # Remove the org link and first few tds, then get next td content
                cleaned_row = re.sub(r'org=[^>]+>.*?</a>', '', row, flags=re.DOTALL | re.IGNORECASE)
                td_matches = re.findall(r'<td[^>]*>([^<]+)</td>', cleaned_row, re.IGNORECASE)
                if td_matches:
                    org_name = td_matches[0].strip()
                else:
                    org_name = ""
            else:
                org_name = name_match.group(1).strip()
            
            # Extract URL - try multiple patterns
            ftp_url = ""
            
            # First priority: GCF genome URLs
            gcf_patterns = [
                r'href="(https?://ftp\.ncbi\.nlm\.nih\.gov/genomes/all/GCF/[^"]*)"',
                r'href="(https?://ftp\.ncbi\.nlm\.nih\.gov/genomes/.*?GCF_[^"]*)"'
            ]
            
            for pattern in gcf_patterns:
                gcf_match = re.search(pattern, row, re.IGNORECASE)
                if gcf_match:
                    ftp_url = gcf_match.group(1)
                    break
            
            # Second priority: RefSeq URLs (but not for hsa, mmu, rno)
            if not ftp_url and org_code not in known_genome_urls:
                refseq_pattern = r'href="(https?://ftp\.ncbi\.nlm\.nih\.gov/refseq/[^"]*)"'
                refseq_match = re.search(refseq_pattern, row, re.IGNORECASE)
                if refseq_match:
                    ftp_url = refseq_match.group(1)
            
            # Third priority: Any other ftp or http URL
            if not ftp_url:
                url_pattern = r'href="(https?://[^"]*)"'
                url_matches = re.findall(url_pattern, row, re.IGNORECASE)
                for url in url_matches:
                    if 'kegg.jp' not in url and ('ftp.' in url or 'http' in url):
                        ftp_url = url
                        break
            
            # Clean the organism name
            org_name = re.sub(r'\s+', ' ', org_name).strip()
            
            # Store the organism
            organisms.append({
                'org_code': org_code,
                'name': org_name,
                'url': ftp_url
            })
            
            # Debug first few organisms
            if organism_count <= 5:
                url_display = ftp_url[:60] + "..." if ftp_url and len(ftp_url) > 60 else ftp_url
                LOG.debug(f"Parsed {org_code}: '{org_name}' -> {url_display or 'NO URL'}")
    
    LOG.info(f"Found {organism_count} organisms with org codes")
    
    # Apply known URL mappings for hsa, mmu, rno
    update_count = 0
    for org in organisms:
        if org['org_code'] in known_genome_urls:
            new_url = known_genome_urls[org['org_code']]
            old_url = org['url']
            
            # Only update if current URL is not already a GCF URL or is empty
            if not old_url or '/refseq/' in old_url or 'mRNA_Prot' in old_url or 'release/complete' in old_url:
                if old_url != new_url:
                    LOG.debug(f"Updating {org['org_code']} URL from {old_url[:50] if old_url else 'None'} to {new_url[:50]}...")
                    org['url'] = new_url
                    update_count += 1
    
    if update_count > 0:
        LOG.info(f"Updated {update_count} organism URLs to known genome references")
    
    return organisms

def verify_organism_urls(organisms: List[Dict]) -> None:
    """Verify that organism codes match their FTP URLs."""
    LOG.info("Verifying organism-URL mappings...")
    
    # Known problematic mappings to check
    problem_organisms = []
    
    for org in organisms[:20]:  # Check first 20
        if org['url']:
            # Check if URL seems to match organism
            url_lower = org['url'].lower()
            name_lower = org['name'].lower()
            org_code = org['org_code']
            
            # Simple heuristic checks
            if org_code == 'hsa' and 'h_sapiens' not in url_lower and 'human' not in url_lower:
                problem_organisms.append((org_code, "Human URL doesn't match"))
            elif org_code == 'ptr' and 'pantro' not in url_lower and 'chimpanzee' not in name_lower:
                problem_organisms.append((org_code, "Chimp URL may not match"))
            elif org_code == 'pps' and 'panpan' not in url_lower and 'bonobo' not in name_lower:
                problem_organisms.append((org_code, "Bonobo URL may not match"))
    
    if problem_organisms:
        LOG.warning(f"Found {len(problem_organisms)} potential URL mismatches:")
        for org_code, reason in problem_organisms[:5]:
            LOG.warning(f"  {org_code}: {reason}")
        if len(problem_organisms) > 5:
            LOG.warning(f"  ... and {len(problem_organisms) - 5} more")
    else:
        LOG.info("All organism-URL mappings look correct")

# OLD MODEL parse_kegg_html()
#
# Due to error(?) at KEGG https://www.kegg.jp/kegg/tables/br08606.html
#   hsa (human) and mmu (house mouse) will link to this
#   hsa: https://ftp.ncbi.nlm.nih.gov/refseq/H_sapiens/mRNA_Prot/
#   mmu: https://ftp.ncbi.nlm.nih.gov/refseq/release/complete/
# Instead of:
#   hsa: https://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/000/001/405/GCF_000001405.40_GRCh38.p14/
#   mmu: https://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/000/001/635/GCF_000001635.27_GRCm39/
# Still we retain this function, in case we are wrong!
#
# def parse_kegg_html(url: str) -> List[Dict]:
    # """
    # Parse KEGG organism HTML table.
    
    # Args:
        # url: URL to KEGG organism HTML page
    
    # Returns:
        # List of organism dictionaries
    # """
    # LOG.info(f"Parsing KEGG HTML from: {url}")
    
    # try:
        # req = urllib.request.Request(
            # url,
            # headers={
                # 'User-Agent': 'Mozilla/5.0 (gpathex/1.0.0)',
                # 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
                # 'Accept-Language': 'en-US,en;q=0.5',
                # 'Accept-Encoding': 'gzip, deflate',
                # 'Connection': 'keep-alive',
                # 'Upgrade-Insecure-Requests': '1'
            # }
        # )
        
        # response = urllib.request.urlopen(req, timeout=30)
        # html_content = response.read().decode('utf-8', errors='ignore')
        
        # # Try simple regex parser first (more reliable for this table)
        # organisms = parse_kegg_html_simple_regex(html_content)
        
        # if organisms:
            # LOG.info(f"Regex parser found {len(organisms)} organisms")
            
            # # Show first few for verification
            # LOG.info("First 5 parsed organisms:")
            # for i, org in enumerate(organisms[:5]):
                # url_preview = org['url'][:60] + "..." if org['url'] and len(org['url']) > 60 else org['url']
                # LOG.info(f"  {i+1}. {org['org_code']}: '{org['name']}' -> {url_preview or 'NO URL'}")
            
            # return organisms
        # else:
            # LOG.warning("Regex parser found no organisms, falling back to HTMLParser")
            
            # # Fallback to HTMLParser
            # parser = KEGGOrgHTMLParser()
            # parser.feed(html_content)
            
            # if parser.organisms:
                # LOG.info(f"HTMLParser found {len(parser.organisms)} organisms")
                # return parser.organisms
            # else:
                # LOG.error("No organisms found with any parser")
                # return []
        
    # except Exception as e:
        # LOG.error(f"Failed to parse HTML from {url}: {e}")
        # return []

def parse_kegg_html(url: str) -> List[Dict]:
    """
    Parse KEGG organism HTML table.
    
    Args:
        url: URL to KEGG organism HTML page
    
    Returns:
        List of organism dictionaries
    """
    LOG.info(f"Parsing KEGG HTML from: {url}")
    
    try:
        req = urllib.request.Request(
            url,
            headers={
                'User-Agent': 'Mozilla/5.0 (gpathex/1.0.0)',
                'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
                'Accept-Language': 'en-US,en;q=0.5',
                'Accept-Encoding': 'gzip, deflate',
                'Connection': 'keep-alive',
                'Upgrade-Insecure-Requests': '1'
            }
        )
        
        response = urllib.request.urlopen(req, timeout=30)
        html_content = response.read().decode('utf-8', errors='ignore')
        
        # Save HTML for debugging if needed
        if LOG.level <= logging.DEBUG:
            debug_file = Path("kegg_html_debug.html")
            with open(debug_file, 'w', encoding='utf-8') as f:
                f.write(html_content)
            LOG.debug(f"Saved HTML to {debug_file} for debugging")
        
        # Try simple regex parser
        organisms = parse_kegg_html_simple_regex(html_content)
        
        if organisms:
            LOG.info(f"Regex parser found {len(organisms)} organisms")
            
            # Show first few for verification
            LOG.info("First 10 parsed organisms:")
            for i, org in enumerate(organisms[:10]):
                url_preview = org['url'][:60] + "..." if org['url'] and len(org['url']) > 60 else org['url']
                LOG.info(f"  {i+1:3}. {org['org_code']}: '{org['name'][:40]}' -> {url_preview or 'NO URL'}")
            
            return organisms
        else:
            LOG.warning("Regex parser found no organisms, falling back to HTMLParser")
            
            # Fallback to HTMLParser
            parser = KEGGOrgHTMLParser()
            parser.feed(html_content)
            
            if parser.organisms:
                LOG.info(f"HTMLParser found {len(parser.organisms)} organisms")
                return parser.organisms
            else:
                LOG.error("No organisms found with any parser")
                return []
        
    except Exception as e:
        LOG.error(f"Failed to parse HTML from {url}: {e}")
        return []

def download_organisms(
    url: str = None,
    output_file: str = None,
    use_api: bool = False,
    **kwargs
) -> int:
    """
    Download KEGG organism information from HTML table.
    
    Args:
        url: KEGG organism list URL (HTML page)
        output_file: Output file path
        use_api: NOT USED - HTML parsing is only method that gives FTP URLs
    
    Returns:
        Exit code (0 = success)
    """
    if output_file is None:
        output_file = config.get("DEFAULT_ORG_FILE")
    
    if url is None:
        url = "https://www.kegg.jp/kegg/tables/br08606.html"
    
    output_path = Path(output_file)
    
    LOG.info(f"Downloading KEGG organism information")
    LOG.info(f"Source URL: {url}")
    LOG.info(f"Output: {output_path}")
    
    if use_api:
        LOG.error("KEGG API mode not implemented in this version")
        LOG.error("Use HTML parsing (default) to get FTP URLs")
        return 1
    
    try:
        # Parse organisms from HTML
        organisms = parse_kegg_html(url)
        
        # Check if we got any organisms
        if not organisms:
            LOG.error(" No organisms found in HTML table")
            LOG.error(" Check the URL or try with --verbose flag to see HTML structure")
            return 1
        
        LOG.info(f" Found {len(organisms)} organisms")
        
        # Count organisms with URLs
        organisms_with_urls = sum(1 for org in organisms if org['url'])
        organisms_without_urls = len(organisms) - organisms_with_urls
        
        LOG.info(f" Organisms with FTP URLs: {organisms_with_urls}/{len(organisms)}")
        
        if organisms_without_urls > 0:
            LOG.warning(f" {organisms_without_urls} organisms without FTP URLs")
            # Log which ones are missing URLs
            for org in organisms[:10]:  # Only show first 10 to avoid spam
                if not org['url']:
                    LOG.debug(f"  No URL for: {org['org_code']} - {org['name']}")
        
        # Save organisms
        if polars_utils.POLARS_AVAILABLE:
            import polars as pl
            
            # Create DataFrame
            df = pl.DataFrame(organisms)
            
            # Save to file
            if output_path.suffix == '.parquet':
                df.write_parquet(output_path)
            elif output_path.suffix == '.csv':
                df.write_csv(output_path)
            else:
                # Default to TSV
                df.write_csv(output_path, separator='\t')
        else:
            # Manual TSV writing
            with open(output_path, 'w', encoding='utf-8') as f:
                f.write("org_code\tname\turl\n")
                for org in organisms:
                    f.write(f"{org['org_code']}\t{org['name']}\t{org['url']}\n")
        
        # Create summary
        summary_path = output_path.with_suffix('.summary.txt')
        with open(summary_path, 'w') as f:
            f.write("KEGG Organisms Summary\n")
            f.write("=" * 50 + "\n")
            f.write(f"Source URL: {url}\n")
            f.write(f"Output file: {output_path}\n")
            f.write(f"Method: HTML parsing\n\n")
            
            f.write(f"Total organisms: {len(organisms)}\n")
            f.write(f"Organisms with FTP URLs: {organisms_with_urls}\n")
            f.write(f"Organisms without URLs: {organisms_without_urls}\n\n")
            
            # List first 20 organisms
            f.write("First 20 organisms:\n")
            f.write("-" * 40 + "\n")
            for i, org in enumerate(organisms[:20]):
                has_url = "✓" if org['url'] else "✗"
                f.write(f"{i+1:3}. {has_url} {org['org_code']}: {org['name']}\n")
                if org['url']:
                    # Truncate long URLs
                    url_display = org['url']
                    if len(url_display) > 100:
                        url_display = url_display[:97] + "..."
                    f.write(f"       URL: {url_display}\n")
        
        # Print results
        print(f"\n✅ Successfully downloaded {len(organisms)} organisms")
        print(f"📁 Saved to: {output_path}")
        print(f"📊 Summary: {summary_path}")
        print(f"🔗 Organisms with FTP URLs: {organisms_with_urls}/{len(organisms)}")
        
        if organisms_with_urls > 0:
            print("\n📋 First 5 organisms:")
            for i, org in enumerate(organisms[:5]):
                if org['url']:
                    # Shorten long URLs for display
                    url_display = org['url']
                    if len(url_display) > 60:
                        url_display = url_display[:57] + "..."
                    print(f"  {i+1}. ✓ {org['org_code']:6} {org['name'][:40]:40} → {url_display}")
                else:
                    print(f"  {i+1}. ✗ {org['org_code']:6} {org['name'][:40]:40} → NO URL")
        
        if organisms_without_urls > 0:
            print(f"\n⚠️  {organisms_without_urls} organisms without FTP URLs")
            print(f"   Check {summary_path} for complete list")
        
        return 0
        
    except Exception as e:
        LOG.error(f" Failed to download organisms: {e}")
        if config.get("LOG_LEVEL") == "DEBUG":
            import traceback
            traceback.print_exc()
        return 1


def main():
    """Command-line entry point."""
    import argparse
    
    parser = argparse.ArgumentParser(
        description="Download KEGG organism information from HTML table (with FTP URLs)"
    )
    parser.add_argument(
        "--url",
        default="https://www.kegg.jp/kegg/tables/br08606.html",
        help="KEGG organism list URL (default: https://www.kegg.jp/kegg/tables/br08606.html)"
    )
    parser.add_argument(
        "--out", "-o",
        default=config.get("DEFAULT_ORG_FILE"),
        help=f"Output file (default: {config.get('DEFAULT_ORG_FILE')})"
    )
    parser.add_argument(
        "--api",
        action="store_true",
        help="(NOT IMPLEMENTED) Use KEGG API instead - gets taxon IDs but NO FTP URLs"
    )
    parser.add_argument(
        "--verbose", "-v",
        action="store_true",
        help="Verbose output (shows HTML parsing details)"
    )
    
    args = parser.parse_args()
    
    # Set up logging
    log_level = "DEBUG" if args.verbose else config.get("LOG_LEVEL")
    logging.basicConfig(
        level=getattr(logging, log_level.upper()),
        format=config.get("LOG_FORMAT")
    )
    
    # Run download
    return download_organisms(
        url=args.url,
        output_file=args.out,
        use_api=args.api
    )


if __name__ == "__main__":
    sys.exit(main())

