import logging
from pathlib import Path
from .config import PipelineConfig
from .directories import ensure_directory
from .index_service import download_index
from .ticker_service import fetch_jse_tickers
from .market_service import download_market_data
from .cleaning import remove_outliers
from .validation import validate_dates

logger = logging.getLogger(__name__)

def run_pipeline(config: PipelineConfig) -> Path:
    validate_dates(config.start, config.end)
    output_dir = ensure_directory(config.output_dir)

    index_df = download_index(config.index_symbol, config.start, config.end)
    index_df.to_csv(output_dir / "index.csv")

    tickers = fetch_jse_tickers(config.api_key)
    market_df = download_market_data(tickers, config.start, config.end)
    cleaned_df = remove_outliers(market_df)

    for symbol, group in cleaned_df.groupby("symbol"):
        group.to_csv(output_dir / f"{symbol}.csv")

    logger.info("Pipeline complete")
    return output_dir
