import cloudscraper
from bs4 import BeautifulSoup
import time
import re
import os
import pandas as pd
import signal
import sys
from pathlib import Path
from urllib.parse import urljoin

class GracefulKiller:
    kill_now = False
    def __init__(self):
        signal.signal(signal.SIGINT, self._kill_handler)
        signal.signal(signal.SIGTERM, self._kill_handler)

    def _kill_handler(self, signum, frame):
        print("\n⚠️  Stopping scraper gracefully... Please wait.")
        self.kill_now = True

def display_categories():
    """Display available categories for user selection"""
    categories = {
        1: ("Development", {
            1: ("Web Developers", "web-developers"),
            2: ("Software Developers", "developers"),
            3: ("Mobile App Development", "app-developers"),
            4: ("eCommerce", "ecommerce-developers"),
            5: ("WordPress Developers", "wordpress"),
            6: ("Shopify", "shopify"),
            7: ("Ruby on Rails", "ruby-rails"),
            8: ("PHP", "php"),
            9: ("Drupal", "drupal"),
            10: ("Magento", "magento")
        }),
        2: ("Marketing", {
            1: ("Digital Marketing", "digital-marketing-agencies"),
            2: ("SEO", "seo-companies"),
            3: ("PPC", "ppc-companies"),
            4: ("Social Media Marketing", "social-media-marketing"),
            5: ("Content Marketing", "content-marketing"),
            6: ("Email Marketing", "email-marketing"),
            7: ("Branding", "branding-agencies"),
            8: ("Public Relations", "pr-firms")
        }),
        3: ("Design", {
            1: ("Web Design", "web-designers"),
            2: ("UX/UI Design", "ux-ui-design"),
            3: ("Graphic Design", "graphic-designers"),
            4: ("Logo Design", "logo-designers"),
            5: ("Product Design", "product-designers")
        }),
        4: ("IT Services", {
            1: ("IT Services", "it-services"),
            2: ("Cybersecurity", "cybersecurity"),
            3: ("Cloud Consulting", "cloud-consulting"),
            4: ("Managed IT Services", "managed-it-services")
        })
    }
    return categories

def get_user_category_selection():
    """Get category selection from user"""
    categories = display_categories()
    
    print("\n" + "="*50)
    print("CLUTCH.CO COMPANY & REVIEWS SCRAPER")
    print("="*50)
    
    # Display main categories
    print("\nSelect a main category:")
    for key, (name, subcats) in categories.items():
        print(f"{key}. {name}")
    
    try:
        main_choice = int(input("\nEnter category number: "))
        if main_choice not in categories:
            print("Invalid selection!")
            return None
            
        main_category, subcategories = categories[main_choice]
        
        # Display subcategories
        print(f"\nSelect from {main_category}:")
        for key, (name, url_slug) in subcategories.items():
            print(f"{key}. {name}")
        
        sub_choice = int(input("\nEnter subcategory number: "))
        if sub_choice not in subcategories:
            print("Invalid selection!")
            return None
            
        selected_name, selected_slug = subcategories[sub_choice]
        return selected_name, selected_slug
        
    except ValueError:
        print("Please enter a valid number!")
        return None

def get_total_companies(scraper, category_url):
    """Get total number of companies in a category"""
    try:
        print(f"\nChecking total companies for: {category_url}")
        response = scraper.get(category_url, timeout=15)
        response.raise_for_status()
        soup = BeautifulSoup(response.text, "html.parser")
        
        # Find total companies count
        companies_count_element = soup.select_one(".facets__companies-amount")
        if companies_count_element:
            count_text = companies_count_element.get_text(strip=True)
            # Extract number from text like "90,154 Companies"
            match = re.search(r'([\d,]+)', count_text)
            if match:
                total_companies = int(match.group(1).replace(',', ''))
                return total_companies, soup
        
        return 0, soup
        
    except Exception as e:
        print(f"Error getting total companies: {e}")
        return 0, None

def scrape_companies_from_page(scraper, page_url):
    """Scrape company information from a single listing page"""
    try:
        print(f"  Scraping companies from: {page_url}")
        response = scraper.get(page_url, timeout=15)
        response.raise_for_status()
        soup = BeautifulSoup(response.text, "html.parser")
        
        companies = []
        
        # Find all company cards
        providers_list = soup.select_one("#providers__list")
        if not providers_list:
            print("    No providers list found")
            return companies
            
        # Look for different types of company listings
        
        # Method 1: Try regular company listings first (non-featured)
        regular_companies = soup.select('.provider-row .provider__title-link[href*="/profile/"]')
        print(f"    Found {len(regular_companies)} regular company profile links")
        
        for link in regular_companies:
            company_name = link.get_text(strip=True)
            company_url = link.get('href')
            
            if company_url.startswith('/'):
                company_url = f"https://clutch.co{company_url}"
            
            companies.append({
                'name': company_name,
                'url': company_url
            })
            print(f"      Found: {company_name}")
        
        # Method 2: If no regular companies found, try alternative selectors
        if not companies:
            print("    No regular listings found, trying alternative selectors...")
            
            # Try different patterns for company profile links
            alternative_selectors = [
                'a[href*="/profile/"]',
                '.provider__title a[href*="/profile/"]',
                '.provider-card a[href*="/profile/"]',
                'h3 a[href*="/profile/"]'
            ]
            
            for selector in alternative_selectors:
                profile_links = soup.select(selector)
                print(f"    Trying selector '{selector}': found {len(profile_links)} links")
                
                for link in profile_links:
                    company_name = link.get_text(strip=True)
                    company_url = link.get('href')
                    
                    if company_url.startswith('/'):
                        company_url = f"https://clutch.co{company_url}"
                    
                    # Avoid duplicates
                    if not any(c['url'] == company_url for c in companies):
                        companies.append({
                            'name': company_name,
                            'url': company_url
                        })
                        print(f"      Found: {company_name}")
                
                if companies:  # If we found some, stop trying other selectors
                    break
        
        # Method 3: Debug - show what company cards we actually have
        if not companies:
            print("    No profile links found. Debugging...")
            all_cards = soup.select('div[id^="provider-"]')
            print(f"    Total provider cards: {len(all_cards)}")
            
            if all_cards:
                first_card = all_cards[0]
                print("    First card HTML snippet:")
                print(f"    {str(first_card)[:500]}...")
                
                # Check all links in first card
                all_links = first_card.select('a')
                print(f"    Links in first card: {len(all_links)}")
                for i, link in enumerate(all_links):
                    href = link.get('href', 'No href')
                    text = link.get_text(strip=True)[:50]
                    print(f"      Link {i+1}: {text} -> {href}")
        
        return companies
        
    except Exception as e:
        print(f"  Error scraping page {page_url}: {e}")
        return []

def save_company_reviews_to_csv(company_data, output_dir):
    """Save company reviews to individual CSV file"""
    company_name = company_data['company_name']
    safe_name = re.sub(r'[^\w\s-]', '', company_name).strip()
    safe_name = re.sub(r'[-\s]+', '_', safe_name)
    
    csv_file = output_dir / f"{safe_name}_reviews.csv"
    
    if company_data['reviews']:
        df = pd.DataFrame(company_data['reviews'])
        df['company_name'] = company_name
        df['company_url'] = company_data['company_url']
        df['scrape_timestamp'] = pd.Timestamp.now()
        
        # Reorder columns
        cols = ['company_name', 'company_url', 'title', 'text', 'reviewer_name', 
                'reviewer_position', 'reviewer_location', 'scrape_timestamp']
        df = df[cols]
        
        df.to_csv(csv_file, index=False, encoding='utf-8')
        print(f"    ✅ Saved {len(df)} reviews to {csv_file.name}")
    else:
        # Create empty CSV with headers for companies with no reviews
        df = pd.DataFrame(columns=['company_name', 'company_url', 'title', 'text', 
                                 'reviewer_name', 'reviewer_position', 'reviewer_location', 'scrape_timestamp'])
        df.to_csv(csv_file, index=False, encoding='utf-8')
        print(f"    ⚠️  No reviews found - created empty CSV: {csv_file.name}")

def save_progress_file(output_dir, scraped_companies, total_requested, category_name):
    """Save progress file to track scraping status"""
    progress_file = output_dir / "scrape_progress.txt"
    with open(progress_file, 'w', encoding='utf-8') as f:
        f.write(f"Category: {category_name}\n")
        f.write(f"Companies scraped: {scraped_companies}\n")
        f.write(f"Total requested: {total_requested}\n")
        f.write(f"Last updated: {time.strftime('%Y-%m-%d %H:%M:%S')}\n")

def scrape_reviews_from_company(scraper, company_url, company_name, killer):
    """Scrape all reviews from a single company profile using original logic"""
    print(f"\n  Scraping reviews for: {company_name}")
    
    all_reviews = []
    
    try:
        # Check for graceful shutdown
        if killer.kill_now:
            return []
            
        # First page
        reviews, soup = scrape_reviews_from_page(scraper, company_url)
        all_reviews.extend(reviews)
        print(f"    Page 1: {len(reviews)} reviews")
        
        # Get company name from profile
        company_element = soup.select_one("h1.profile-header__title")
        actual_company_name = company_element.get_text(strip=True) if company_element else company_name
        
        # Pagination: find total reviews and calculate pages
        pagination_info = soup.select_one("#reviews-list > div.profile-reviews--pagination > p")
        total_pages = 1
        total_reviews = len(reviews)
        
        if pagination_info:
            text = pagination_info.get_text(strip=True)
            print(f"    Pagination info: {text}")
            match = re.search(r'of\s+(\d+)', text, re.IGNORECASE)
            if match:
                total_reviews = int(match.group(1))
                total_pages = (total_reviews + 9) // 10  # assume 10 reviews per page
                print(f"    Total reviews: {total_reviews}, Estimated pages: {total_pages}")
        
        # Scrape remaining pages using actual pagination URLs
        pagination_container = soup.select_one(".sg-pagination-v2")
        page_links = []
        
        if pagination_container and total_pages > 1:
            # Extract all pagination links
            links = pagination_container.select("a[data-page]")
            for link in links:
                href = link.get('href')
                page_num = link.get_text(strip=True)
                if href and page_num.isdigit() and int(page_num) > 1:
                    full_url = f"https://clutch.co{href}"
                    page_links.append((int(page_num), full_url))
            
            page_links = sorted(page_links)
            print(f"    Found {len(page_links)} additional pages to scrape")
            
            for page_num, page_url in page_links:
                # Check for graceful shutdown
                if killer.kill_now:
                    print("    Stopping page scraping due to user interrupt...")
                    break
                    
                if len(all_reviews) >= total_reviews:
                    print(f"    Reached expected total of {total_reviews} reviews!")
                    break
                    
                print(f"    Scraping page {page_num}...")
                try:
                    page_reviews, _ = scrape_reviews_from_page(scraper, page_url)
                    if page_reviews:
                        all_reviews.extend(page_reviews)
                        print(f"    Added {len(page_reviews)} reviews from page {page_num}")
                        print(f"    Total so far: {len(all_reviews)}/{total_reviews}")
                    time.sleep(2)
                except Exception as e:
                    print(f"    Error scraping page {page_num}: {e}")
                    continue
        
        print(f"    Final total reviews for {actual_company_name}: {len(all_reviews)}")
        return all_reviews
        
    except Exception as e:
        print(f"  Error scraping reviews for {company_name}: {e}")
        return []

def scrape_reviews_from_page(scraper, url):
    """Scrape reviews from a single page (reusing existing function)"""
    response = scraper.get(url, timeout=15)
    response.raise_for_status()
    soup = BeautifulSoup(response.text, "html.parser")
    
    reviews = []
    reviews_container_list = soup.select("article.profile-review")
    
    if not reviews_container_list:
        reviews_container_list = soup.select("#reviews-list .profile-review, .review-card, [class*='review'][class*='card']")
    
    if len(reviews_container_list) < 10:
        div_reviews = soup.select("#reviews-list div[id^='review']")
        if div_reviews:
            for div_review in div_reviews:
                if div_review not in reviews_container_list:
                    reviews_container_list.append(div_review)
    
    for review in reviews_container_list:
        title_element = review.select_one(".profile-review__quote") or review.select_one("h3[itemprop='name']") or review.select_one("h3")
        title = title_element.get_text(strip=True) if title_element else ""
        
        text_element = review.select_one(".profile-review__text") or review.select_one("[itemprop='reviewBody']") or review.select_one("p")
        text = text_element.get_text(strip=True) if text_element else ""
        
        reviewer_name = review.select_one(".reviewer_card--name") or review.select_one("[itemprop='author']")
        reviewer_name = reviewer_name.get_text(strip=True) if reviewer_name else ""
        
        reviewer_position = review.select_one(".reviewer_card--position") or review.select_one(".reviewer_position")
        reviewer_position = reviewer_position.get_text(strip=True) if reviewer_position else ""
        
        reviewer_location = review.select_one(
            ".reviewer_card--location, li:nth-child(2) > span.reviewer_list__details-title.sg-text__title"
        )
        reviewer_location = reviewer_location.get_text(strip=True) if reviewer_location else ""

        if (text and len(text) > 50) and title and title not in ["Overall Review Rating", "No title"]:
            reviews.append({
                "title": title,
                "text": text,
                "reviewer_name": reviewer_name if reviewer_name else "Anonymous",
                "reviewer_position": reviewer_position if reviewer_position else "No position",
                "reviewer_location": reviewer_location if reviewer_location else "No location"
            })
    
    return reviews, soup

def main():
    """Main function to run the scraper"""
    killer = GracefulKiller()
    scraper = cloudscraper.create_scraper()
    
    print("💡 Tip: Press Ctrl+C at any time to stop and save progress")
    
    # Get category selection
    selection = get_user_category_selection()
    if not selection:
        print("Invalid selection. Exiting...")
        return
    
    category_name, category_slug = selection
    category_url = f"https://clutch.co/{category_slug}"
    
    # Get total companies
    total_companies, soup = get_total_companies(scraper, category_url)
    if total_companies == 0:
        print("Could not find companies in this category. Exiting...")
        return
    
    print(f"\nCategory: {category_name}")
    print(f"Total companies available: {total_companies:,}")
    
    # Get user input for number of companies
    try:
        num_companies = int(input(f"\nHow many companies do you want to scrape? (max {total_companies:,}): "))
        if num_companies <= 0 or num_companies > total_companies:
            print(f"Please enter a number between 1 and {total_companies:,}")
            return
    except ValueError:
        print("Please enter a valid number!")
        return
    
    # Create output directory
    timestamp = time.strftime("%Y%m%d_%H%M%S")
    safe_category = re.sub(r'[^\w\s-]', '', category_name).strip()
    safe_category = re.sub(r'[-\s]+', '_', safe_category)
    output_dir = Path(f"{safe_category}_{num_companies}_companies_{timestamp}")
    output_dir.mkdir(exist_ok=True)
    
    print(f"\n📁 Output directory: {output_dir}")
    print(f"🚀 Starting to scrape {num_companies} companies and all their reviews...")
    print("This may take a while depending on the number of companies and reviews...")
    
    # Start scraping companies
    scraped_companies = 0
    current_page = 0
    
    try:
        while scraped_companies < num_companies and not killer.kill_now:
            # Build page URL
            if current_page == 0:
                page_url = category_url
            else:
                page_url = f"{category_url}?page={current_page}"
            
            print(f"\nScraping company listing page {current_page + 1}...")
            
            # Get companies from this page
            page_companies = scrape_companies_from_page(scraper, page_url)
            
            if not page_companies:
                print(f"No companies found on page {current_page + 1}. Ending scrape.")
                break
            
            # Process each company until we reach the target
            for company in page_companies:
                if scraped_companies >= num_companies or killer.kill_now:
                    break
                
                company_name = company['name']
                company_url = company['url']
                
                print(f"\n[{scraped_companies + 1}/{num_companies}] Processing: {company_name}")
                
                # Scrape all reviews for this company
                reviews = scrape_reviews_from_company(scraper, company_url, company_name, killer)
                
                company_data = {
                    'company_name': company_name,
                    'company_url': company_url,
                    'reviews': reviews,
                    'total_reviews': len(reviews)
                }
                
                # Save immediately to CSV
                save_company_reviews_to_csv(company_data, output_dir)
                
                scraped_companies += 1
                
                # Save progress
                save_progress_file(output_dir, scraped_companies, num_companies, category_name)
                
                if killer.kill_now:
                    print("Stopping due to user interrupt...")
                    break
                
                # Add delay between companies
                time.sleep(3)
            
            current_page += 1
    
    except KeyboardInterrupt:
        print("\n⚠️  Interrupted by user")
    except Exception as e:
        print(f"\n❌ Error during scraping: {e}")
    
    # Final summary
    print(f"\n" + "="*50)
    if killer.kill_now:
        print("SCRAPING STOPPED BY USER")
    else:
        print("SCRAPING COMPLETED!")
    
    print(f"Companies processed: {scraped_companies}/{num_companies}")
    print(f"Output directory: {output_dir}")
    print(f"CSV files created: {len(list(output_dir.glob('*_reviews.csv')))}")
    print("="*50)

if __name__ == "__main__":
    main()