"""MLB Odds Scraper for retrieving odds data from OddsPortal."""

import pandas as pd
import datetime
from selenium import webdriver
from stealthenium import stealth
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from concurrent.futures import ThreadPoolExecutor
import time
import statsapi

def scrape_page(year, page_num):
    """
    Scrape a single page of MLB odds data.
    
    Args:
        year (int): Year to scrape data for
        page_num (int): Page number to scrape
        
    Returns:
        list: List of dictionaries containing game data
    """
    options = webdriver.ChromeOptions()
    options.add_argument("--headless")
    options.add_argument("start-maximized") 
    options.add_argument("--disable-blink-features=AutomationControlled")
    options.add_experimental_option("excludeSwitches", ["enable-automation"])
    options.add_experimental_option('useAutomationExtension', False)

    driver = webdriver.Chrome(options=options)
    
    stealth(driver,
        user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36',
        languages=["en-US", "en"],
        vendor="Google Inc.",
        platform="Win32",
        webgl_vendor="Intel Inc.",
        renderer="Intel Iris OpenGL Engine",
        fix_hairline=True,
    )
    
    games = []
    
    # For current year, use URL without year suffix
    current_year = datetime.datetime.now().year
    if year == current_year:
        url = f"https://www.oddsportal.com/baseball/usa/mlb/results/#/page/{page_num}/"
    else:
        url = f"https://www.oddsportal.com/baseball/usa/mlb-{year}/results/#/page/{page_num}/"
        
    print(f"Scraping {url}")
    
    try:
        driver.get(url)
        
        # Handle cookie consent if it appears
        try:
            WebDriverWait(driver, 10).until(
                EC.element_to_be_clickable((By.ID, "onetrust-reject-all-handler"))
            ).click()
            print("Clicked cookie reject button")
        except:
            try:
                WebDriverWait(driver, 5).until(
                    EC.element_to_be_clickable((By.XPATH, "//button[contains(text(), 'Reject')]"))
                ).click()
                print("Clicked alternative cookie reject button")
            except:
                print("No cookie consent dialog found or couldn't be clicked")
        
        # Wait for page to load - reduced from 5 to 2 seconds
        time.sleep(2)
        
        # Try different selectors for event rows
        selectors = [
            "div.eventRow",
            "div[data-v-b8d70024].eventRow",
            "div[data-v-b8d70024][id][set]",
            "div.event__match",
            "div.table-main__tr",
            "div.result-table__row"
        ]
        
        events = []
        for selector in selectors:
            try:
                print(f"Trying selector: {selector}")
                WebDriverWait(driver, 5).until(
                    EC.presence_of_element_located((By.CSS_SELECTOR, selector))
                )
                events = driver.find_elements(By.CSS_SELECTOR, selector)
                if events:
                    print(f"Found {len(events)} events with selector: {selector}")
                    break
            except Exception as e:
                print(f"Selector {selector} failed: {str(e)}")
        
        # If no events found with CSS selectors, try XPath
        if not events:
            try:
                print("Trying XPath selector")
                events = driver.find_elements(By.XPATH, "//div[contains(@class, 'event') and .//div[contains(@class, 'participant')]]")
                print(f"Found {len(events)} events with XPath selector")
            except Exception as e:
                print(f"XPath selector failed: {str(e)}")
        
        # Scroll to load all content - reduced from 2 to 1 second
        driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
        time.sleep(1)
        
        print(f"Found {len(events)} events on page {page_num}")
        current_date = None
        
        # Find initial date
        date_selectors = [
            ".text-black-main",
            ".event__time",
            ".table-main__date",
            ".date"
        ]
        
        for event in events:
            for date_selector in date_selectors:
                try:
                    date_elements = event.find_elements(By.CSS_SELECTOR, date_selector)
                    for date_element in date_elements:
                        date_text = date_element.text
                        if len(date_text.split()) >= 3 and any(yr in date_text for yr in [str(year)]) and "Baseball" not in date_text:
                            if " - " in date_text:
                                current_date = date_text.split(' - ')[0].strip()
                            else:
                                current_date = date_text.strip()
                            break
                    if current_date:
                        break
                except:
                    continue
            if current_date:
                break
                
        # Process events
        for event in events:
            try:
                # Try to get date for this event
                for date_selector in date_selectors:
                    try:
                        date_elements = event.find_elements(By.CSS_SELECTOR, date_selector)
                        for date_element in date_elements:
                            date_text = date_element.text
                            if len(date_text.split()) >= 3 and any(yr in date_text for yr in [str(year)]) and "Baseball" not in date_text:
                                if " - " in date_text:
                                    current_date = date_text.split(' - ')[0].strip()
                                else:
                                    current_date = date_text.strip()
                                break
                    except:
                        continue
                
                # Try different selectors for team names
                team_selectors = [
                    ".participant-name",
                    "p.participant-name",
                    ".event__participant",
                    ".table-main__participant",
                    ".participant"
                ]
                
                teams = []
                for team_selector in team_selectors:
                    try:
                        teams = event.find_elements(By.CSS_SELECTOR, team_selector)
                        if len(teams) >= 2:
                            break
                    except:
                        continue
                
                # Try different selectors for scores
                score_selectors = [
                    "div.text-gray-dark div.font-bold",
                    ".font-bold",
                    ".event__score",
                    ".table-main__score",
                    ".score"
                ]
                
                home_score = ''
                away_score = ''
                for score_selector in score_selectors:
                    try:
                        score_section = event.find_element(By.CSS_SELECTOR, score_selector)
                        score_text = score_section.text
                        if '–' in score_text:
                            scores = score_text.split('–')
                            home_score = scores[0].strip()
                            away_score = scores[1].strip()
                            break
                        elif ':' in score_text and score_text[0].isdigit():
                            scores = score_text.split(':')
                            home_score = scores[0].strip()
                            away_score = scores[1].strip()
                            break
                    except:
                        try:
                            score_elements = event.find_elements(By.CSS_SELECTOR, score_selector)
                            if len(score_elements) >= 2:
                                home_score = score_elements[0].text.strip()
                                away_score = score_elements[1].text.strip()
                                break
                        except:
                            continue
                    
                if len(teams) < 2:
                    continue

                # Try different selectors for time
                time_selectors = [
                    ".//p[contains(text(), ':')]",
                    ".//span[contains(text(), ':')]",
                    ".//div[contains(text(), ':') and string-length(.) <= 8]"
                ]
                
                game_time = None
                for time_selector in time_selectors:
                    try:
                        time_element = event.find_element(By.XPATH, time_selector)
                        time_text = time_element.text
                        if ':' in time_text and len(time_text) <= 8:
                            game_time = time_text
                            break
                    except:
                        continue
                
                if not game_time:
                    continue

                home_team = teams[0].text
                away_team = teams[1].text
                    
                # Try different selectors for odds
                odds_selectors = [
                    "[data-testid='add-to-coupon-button'] .height-content",
                    ".gradient-green-added-border",
                    ".event__odd",
                    ".table-main__odd",
                    ".odd"
                ]
                
                odds_elements = []
                for odds_selector in odds_selectors:
                    try:
                        odds_elements = event.find_elements(By.CSS_SELECTOR, odds_selector)
                        if len(odds_elements) >= 2:
                            break
                    except:
                        continue

                if len(odds_elements) >= 2:
                    home_odds = odds_elements[0].text
                    away_odds = odds_elements[1].text

                    if all([current_date, game_time, away_team, home_team, away_odds, home_odds]):
                        game_data = {
                            'date': current_date,
                            'time': game_time,
                            'away_team': away_team,
                            'home_team': home_team,
                            'away_odds': away_odds,
                            'home_odds': home_odds,
                            'away_score': away_score,
                            'home_score': home_score
                        }
                        games.append(game_data)
                
            except Exception as e:
                print(f"Error processing event: {e}")
                continue
        
    except Exception as e:
        print(f"Error scraping page {page_num} for year {year}: {e}")
    finally:
        driver.quit()
        
    return games

def scrape_future_matches():
    """
    Scrape upcoming MLB matches from the main MLB page.
    
    Returns:
        list: List of dictionaries containing future game data
    """
    options = webdriver.ChromeOptions()
    options.add_argument("--headless")
    options.add_argument("start-maximized") 
    options.add_argument("--disable-blink-features=AutomationControlled")
    options.add_experimental_option("excludeSwitches", ["enable-automation"])
    options.add_experimental_option('useAutomationExtension', False)

    driver = webdriver.Chrome(options=options)
    
    stealth(driver,
        user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36',
        languages=["en-US", "en"],
        vendor="Google Inc.",
        platform="Win32",
        webgl_vendor="Intel Inc.",
        renderer="Intel Iris OpenGL Engine",
        fix_hairline=True,
    )
    
    future_games = []
    url = "https://www.oddsportal.com/baseball/usa/mlb/"
    
    print(f"Scraping future matches from {url}")
    
    try:
        driver.get(url)
        
        # Handle cookie consent if it appears
        try:
            WebDriverWait(driver, 10).until(
                EC.element_to_be_clickable((By.ID, "onetrust-reject-all-handler"))
            ).click()
            print("Clicked cookie reject button")
        except:
            try:
                WebDriverWait(driver, 5).until(
                    EC.element_to_be_clickable((By.XPATH, "//button[contains(text(), 'Reject')]"))
                ).click()
                print("Clicked alternative cookie reject button")
            except:
                print("No cookie consent dialog found or couldn't be clicked")
        
        # Wait for page to load
        time.sleep(2)
        
        # Try different selectors for future events
        selectors = [
            "div.eventRow",
            "div[data-v-b8d70024].eventRow",
            "div.event__match",
            ".upcoming-event",
            "div.table-main__tr"
        ]
        
        events = []
        for selector in selectors:
            try:
                print(f"Trying selector for future events: {selector}")
                WebDriverWait(driver, 5).until(
                    EC.presence_of_element_located((By.CSS_SELECTOR, selector))
                )
                events = driver.find_elements(By.CSS_SELECTOR, selector)
                if events:
                    print(f"Found {len(events)} future events with selector: {selector}")
                    break
            except Exception as e:
                print(f"Future selector {selector} failed: {str(e)}")
        
        # If no events found with CSS selectors, try XPath
        if not events:
            try:
                print("Trying XPath selector for future events")
                events = driver.find_elements(By.XPATH, "//div[contains(@class, 'event') and .//div[contains(@class, 'participant')]]")
                print(f"Found {len(events)} future events with XPath selector")
            except Exception as e:
                print(f"Future XPath selector failed: {str(e)}")
        
        # Scroll to load all content
        driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
        time.sleep(1)
        
        # Process future events
        current_date = None
        for event in events:
            try:
                # Try to get date for this event
                date_selectors = [
                    ".text-black-main",
                    ".event__time",
                    ".table-main__date",
                    ".date",
                    ".event-time"
                ]
                
                for date_selector in date_selectors:
                    try:
                        date_elements = event.find_elements(By.CSS_SELECTOR, date_selector)
                        for date_element in date_elements:
                            date_text = date_element.text
                            # For future events, date might be formatted differently
                            if len(date_text.split()) >= 2 and "Baseball" not in date_text:
                                current_date = date_text.strip()
                                break
                        if current_date:
                            break
                    except:
                        continue
                
                # Try different selectors for team names
                team_selectors = [
                    ".participant-name",
                    "p.participant-name",
                    ".event__participant",
                    ".table-main__participant",
                    ".participant"
                ]
                
                teams = []
                for team_selector in team_selectors:
                    try:
                        teams = event.find_elements(By.CSS_SELECTOR, team_selector)
                        if len(teams) >= 2:
                            break
                    except:
                        continue
                
                if len(teams) < 2:
                    continue

                # Try different selectors for time
                time_selectors = [
                    ".//p[contains(text(), ':')]",
                    ".//span[contains(text(), ':')]",
                    ".//div[contains(text(), ':') and string-length(.) <= 8]",
                    ".event-time"
                ]
                
                game_time = None
                for time_selector in time_selectors:
                    try:
                        time_element = event.find_element(By.XPATH, time_selector)
                        time_text = time_element.text
                        if ':' in time_text and len(time_text) <= 8:
                            game_time = time_text
                            break
                    except:
                        continue
                
                if not game_time:
                    # Try to extract time from date if it's combined
                    if current_date and ':' in current_date:
                        parts = current_date.split()
                        for part in parts:
                            if ':' in part and len(part) <= 8:
                                game_time = part
                                # Remove time from date
                                current_date = ' '.join([p for p in parts if p != part])
                                break

                if not game_time:
                    continue

                home_team = teams[0].text
                away_team = teams[1].text
                    
                # Try different selectors for odds
                odds_selectors = [
                    "[data-testid='add-to-coupon-button'] .height-content",
                    ".gradient-green-added-border",
                    ".event__odd",
                    ".table-main__odd",
                    ".odd"
                ]
                
                odds_elements = []
                for odds_selector in odds_selectors:
                    try:
                        odds_elements = event.find_elements(By.CSS_SELECTOR, odds_selector)
                        if len(odds_elements) >= 2:
                            break
                    except:
                        continue

                home_odds = ''
                away_odds = ''
                if len(odds_elements) >= 2:
                    home_odds = odds_elements[0].text
                    away_odds = odds_elements[1].text

                if all([current_date, game_time, away_team, home_team]):
                    # For future games, scores are not available
                    game_data = {
                        'date': current_date,
                        'time': game_time,
                        'away_team': away_team,
                        'home_team': home_team,
                        'away_odds': away_odds,
                        'home_odds': home_odds,
                        'away_score': '',  # Empty for future games
                        'home_score': '',  # Empty for future games
                        'is_future': True  # Flag to identify future games
                    }
                    future_games.append(game_data)
                
            except Exception as e:
                print(f"Error processing future event: {e}")
                continue
        
    except Exception as e:
        print(f"Error scraping future matches: {e}")
    finally:
        driver.quit()
        
    print(f"Total future games collected: {len(future_games)}")
    return future_games

def scrape_oddsportal_mlb(year, max_pages=None, include_future=True):
    """
    Scrape MLB odds data for a specific year.
    
    Args:
        year (int): Year to scrape data for
        max_pages (int, optional): Maximum number of pages to scrape
        include_future (bool, optional): Whether to include future matches
        
    Returns:
        list: List of dictionaries containing game data
    """
    # Get total pages
    options = webdriver.ChromeOptions()
    options.add_argument("--headless")
    driver = webdriver.Chrome(options=options)
    
    total_pages = 1
    try:
        # For current year, use URL without year suffix
        current_year = datetime.datetime.now().year
        if year == current_year:
            url = f"https://www.oddsportal.com/baseball/usa/mlb/results/#/page/1/"
        else:
            url = f"https://www.oddsportal.com/baseball/usa/mlb-{year}/results/#/page/1/"
            
        print(f"Getting pagination info from {url}")
        driver.get(url)
        
        # Handle cookie consent if it appears
        try:
            WebDriverWait(driver, 10).until(
                EC.element_to_be_clickable((By.ID, "onetrust-reject-all-handler"))
            ).click()
        except:
            try:
                WebDriverWait(driver, 5).until(
                    EC.element_to_be_clickable((By.XPATH, "//button[contains(text(), 'Reject')]"))
                ).click()
            except:
                print("No cookie consent dialog found or couldn't be clicked")
        
        # Wait for page to load - reduced from 5 to 2 seconds
        time.sleep(2)
        
        # Try different selectors for pagination
        pagination_selectors = [
            "div.pagination",
            "ul.pagination",
            ".pagination-container",
            ".pagination-wrapper"
        ]
        
        for selector in pagination_selectors:
            try:
                pagination = WebDriverWait(driver, 10).until(
                    EC.presence_of_element_located((By.CSS_SELECTOR, selector))
                )
                page_links = pagination.find_elements(By.CSS_SELECTOR, "a.pagination-link, a[data-number]")
                
                if page_links:
                    page_numbers = []
                    for link in page_links:
                        data_number = link.get_attribute('data-number')
                        if data_number and data_number.isdigit():
                            page_numbers.append(int(data_number))
                        elif link.text and link.text.isdigit():
                            page_numbers.append(int(link.text))
                    
                    if page_numbers:
                        total_pages = max(page_numbers)
                        break
            except:
                continue
        
        # If pagination not found, check if there's any indication of multiple pages
        if total_pages == 1:
            try:
                next_page = driver.find_element(By.XPATH, "//a[contains(text(), 'Next') or contains(@class, 'next')]")
                if next_page:
                    total_pages = 2  # At least 2 pages
            except:
                pass
    except Exception as e:
        print(f"Error determining total pages for year {year}: {e}")
    finally:
        driver.quit()
    
    if max_pages:
        total_pages = min(max_pages, total_pages)
    
    print(f"Will scrape {total_pages} pages")
    
    # Scrape pages concurrently - increase workers from 5 to 8
    all_games = []
    with ThreadPoolExecutor(max_workers=8) as executor:
        futures = [executor.submit(scrape_page, year, page) for page in range(1, total_pages + 1)]
        for future in futures:
            all_games.extend(future.result())
    
    # Add future games if requested and if we're scraping the current year
    if include_future and year == datetime.datetime.now().year:
        future_games = scrape_future_matches()
        if future_games:
            all_games.extend(future_games)
            
    print(f"Total games collected: {len(all_games)}")
    return all_games

def scrape_oddsportal_mlb_years(start_year=2006, end_year=2025, max_pages=None, include_future=True):
    """
    Scrape MLB odds data for multiple years.
    
    Args:
        start_year (int): First year to scrape (inclusive)
        end_year (int): Last year to scrape (exclusive)
        max_pages (int, optional): Maximum number of pages to scrape per year
        include_future (bool, optional): Whether to include future matches
        
    Returns:
        pd.DataFrame: DataFrame containing all scraped games
    """
    all_games = []
    for year in range(start_year, end_year):
        print(f"\nScraping year {year}...")
        games = scrape_oddsportal_mlb(year, max_pages, include_future)
        
        if games:
            all_games.extend(games)
        else:
            print(f"No games found for year {year}")
        
    return pd.DataFrame(all_games) if all_games else pd.DataFrame()

def process_game_data(df):
    """
    Process raw MLB game data by cleaning and transforming fields.
    
    Args:
        df (pd.DataFrame): Raw dataframe containing MLB game data
        
    Returns:
        pd.DataFrame: Processed dataframe with team IDs in 'home_team' and 'away_team' columns.
    """
    if df.empty:
        print("DataFrame is empty, not processing")
        return df
    
    processed_df = df.copy()
    
    # Mark which games are future games if the column exists
    if 'is_future' not in processed_df.columns:
        processed_df['is_future'] = False
    
    # Ensure is_future is boolean, not float
    processed_df['is_future'] = processed_df['is_future'].fillna(False).astype(bool)
    
    # Convert date to datetime, handling future dates differently
    try:
        # Handle different date formats for historical and future games
        future_mask = processed_df['is_future'] == True
        if not future_mask.empty:
            # For historical games
            if (~future_mask).any():
                processed_df.loc[~future_mask, 'game_date'] = pd.to_datetime(processed_df.loc[~future_mask, 'date'])
                processed_df.loc[~future_mask, 'game_datetime'] = pd.to_datetime(
                    processed_df.loc[~future_mask, 'date'] + ' ' + processed_df.loc[~future_mask, 'time']
                )
            
            # For future games with different date format
            if future_mask.any():
                # Handle date strings like "Today, 28 Mar", "Tomorrow, 29 Mar", or "Wednesday, 30 Mar"
                future_dates = []
                future_datetimes = []
                today = pd.Timestamp.now().normalize()
                
                for idx, row in processed_df[future_mask].iterrows():
                    date_str = row['date']
                    time_str = row['time']
                    
                    try:
                        if date_str.startswith('Today'):
                            game_date = today
                        elif date_str.startswith('Tomorrow'):
                            game_date = today + pd.Timedelta(days=1)
                        elif date_str.startswith('Yesterday'):
                            game_date = today - pd.Timedelta(days=1)
                        else:
                            # Try to extract date parts
                            date_parts = date_str.split(',')
                            if len(date_parts) >= 2:
                                # Format like "Wednesday, 30 Mar"
                                day_month = date_parts[1].strip()
                                # Extract day and month, add current year
                                day = int(day_month.split()[0])
                                month_abbr = day_month.split()[1]
                                month_map = {
                                    'Jan': 1, 'Feb': 2, 'Mar': 3, 'Apr': 4,
                                    'May': 5, 'Jun': 6, 'Jul': 7, 'Aug': 8,
                                    'Sep': 9, 'Oct': 10, 'Nov': 11, 'Dec': 12
                                }
                                month = month_map.get(month_abbr, 1)
                                year = today.year
                                game_date = pd.Timestamp(year, month, day)
                            else:
                                # Fallback if can't parse
                                game_date = today
                        
                        # Create game_datetime by combining date and time
                        hour, minute = map(int, time_str.split(':'))
                        game_datetime = game_date.replace(hour=hour, minute=minute)
                        
                        future_dates.append(game_date)
                        future_datetimes.append(game_datetime)
                    except Exception as e:
                        print(f"Error parsing future date '{date_str}': {e}")
                        future_dates.append(today)
                        future_datetimes.append(today)
                
                # Assign back to dataframe
                processed_df.loc[future_mask, 'game_date'] = future_dates
                processed_df.loc[future_mask, 'game_datetime'] = future_datetimes
        else:
            # No future games, process all as historical
            processed_df['game_date'] = pd.to_datetime(processed_df['date'])
            processed_df['game_datetime'] = pd.to_datetime(processed_df['date'] + ' ' + processed_df['time'])
        
        # Try to localize to Eastern time and convert to UTC
        try:
            # Only localize non-NaT values
            valid_dt_mask = ~pd.isna(processed_df['game_datetime'])
            if valid_dt_mask.any():
                processed_df.loc[valid_dt_mask, 'game_datetime'] = (
                    processed_df.loc[valid_dt_mask, 'game_datetime']
                    .dt.tz_localize('US/Eastern', ambiguous='raise')
                    .dt.tz_convert('UTC')
                )
        except Exception as e:
            print(f"Could not convert datetime to UTC: {e}")
            print("Using naive datetime")
    except Exception as e:
        print(f"Error converting dates: {e}")
        processed_df['game_date'] = processed_df['date']
        processed_df['game_datetime'] = processed_df['date'] + ' ' + processed_df['time']
    
    # Drop rows containing All Star Game matchups
    processed_df = processed_df[~(processed_df['away_team'].isin(['American League', 'National League']) | 
                                processed_df['home_team'].isin(['American League', 'National League']))]
    
    # Clean team names and store original abbreviations
    processed_df['home_team_abbr'] = processed_df['home_team'].str.replace('St.Louis', 'St. Louis')
    processed_df['away_team_abbr'] = processed_df['away_team'].str.replace('St.Louis', 'St. Louis')
    
    # Get unique team names
    all_teams = processed_df['away_team_abbr'].tolist() + processed_df['home_team_abbr'].tolist()
    teams = []
    for team in all_teams:
        if team not in teams:
            teams.append(team)

    # Create mapping of team names to IDs
    team_mapping = {}
    for team in teams:
        try:
            team_info = statsapi.lookup_team(team)
            if team_info:
                team_mapping[team] = team_info[0]['id']
            else:
                print(f"Could not find ID for team: {team}")
                team_mapping[team] = None
        except Exception as e:
            print(f"Error looking up team {team}: {e}")
            team_mapping[team] = None

    # Map team names to IDs in new columns
    processed_df['home_team_id_mapped'] = processed_df['home_team_abbr'].map(team_mapping)
    processed_df['away_team_id_mapped'] = processed_df['away_team_abbr'].map(team_mapping)

    # Overwrite original team columns with IDs and rename
    processed_df['home_team'] = processed_df['home_team_id_mapped']
    processed_df['away_team'] = processed_df['away_team_id_mapped']
    
    # Drop the temporary ID columns
    processed_df.drop(columns=['home_team_id_mapped', 'away_team_id_mapped'], inplace=True)

    # Convert odds columns - replace + with empty string and convert to numeric
    try:
        processed_df['away_odds'] = pd.to_numeric(processed_df['away_odds'].str.replace('+', ''), errors='coerce')
        processed_df['home_odds'] = pd.to_numeric(processed_df['home_odds'].str.replace('+', ''), errors='coerce')
    except Exception as e:
        print(f"Error converting odds: {e}")

    # Convert score columns to numeric - only for non-future games
    try:
        # Only convert scores for past games
        future_mask = processed_df['is_future'] == True
        non_future_mask = ~future_mask
        if non_future_mask.any():
            processed_df.loc[non_future_mask, 'away_score'] = pd.to_numeric(processed_df.loc[non_future_mask, 'away_score'], errors='coerce')
            processed_df.loc[non_future_mask, 'home_score'] = pd.to_numeric(processed_df.loc[non_future_mask, 'home_score'], errors='coerce')
    except Exception as e:
        print(f"Error converting scores: {e}")
    
    # Drop rows with missing scores if they exist - but only for past games, not future ones
    if 'away_score' in processed_df.columns and 'home_score' in processed_df.columns:
        future_mask = processed_df['is_future'] == True
        non_future_mask = ~future_mask
        if non_future_mask.any():
            # Only filter out missing scores for non-future games
            missing_scores = processed_df.loc[non_future_mask, ['home_score', 'away_score']].isna().any(axis=1)
            drop_indices = processed_df.loc[non_future_mask][missing_scores].index
            processed_df = processed_df.drop(drop_indices)
    
    # Select final columns - now home_team and away_team contain IDs
    final_columns = ['game_date', 'game_datetime', 'home_team', 'away_team', 
                     'home_odds', 'away_odds', 'home_score', 'away_score', 
                     'home_team_abbr', 'away_team_abbr']
    
    # Only keep columns that exist
    final_columns = [col for col in final_columns if col in processed_df.columns]
    processed_df = processed_df[final_columns]
    
    return processed_df 