import os
from pathlib import Path
import pandas as pd

# Desired order for Models
MODEL_ORDER = ["ANN", "DecisionTree", "LinearRegression", "RandomForest", "XGBoost"]

def read_csv_any(path):
    """Try multiple encodings until one works."""
    encodings = ["utf-8", "utf-8-sig", "latin1", "ISO-8859-1", "cp1252"]
    for enc in encodings:
        try:
            return pd.read_csv(path, encoding=enc), enc
        except Exception:
            continue
    return None, None

def combine(FOLDER, OUTPUT, REQUIRED_COLS):
    all_dfs = []
    for f in sorted(FOLDER.iterdir()):
        if not f.is_file():
            continue
        df, enc = read_csv_any(f)
        if df is None or df.empty:
            print(f"⚠️ Skipping {f.name} (unreadable or empty)")
            continue

        # Drop Unnamed columns
        df = df.loc[:, ~df.columns.str.startswith("Unnamed")]

        if not REQUIRED_COLS.issubset(df.columns):
            print(f"⚠️ Skipping {f.name} (missing columns)")
            continue

        all_dfs.append(df)

    if not all_dfs:
        print("❌ No valid files found.")
        return

    combined = pd.concat(all_dfs, ignore_index=True)

    # Sort so same dataset stay together + model order enforced
    # combined["Model"] = pd.Categorical(combined["Model"], categories=MODEL_ORDER, ordered=True)
    combined = combined.sort_values(by=["Dataset", "Model", "Method"], ignore_index=True)

    combined.to_csv(OUTPUT, index=False)
    print(f"✅ Combined & sorted file saved as {OUTPUT}")

if __name__ == "__main__":
    FOLDER = Path("sert&sera")
    OUTPUT = Path("SERT&SERA.csv")
    REQUIRED_COLS = {"Dataset", "Model", "Method", "Raw SERA", "Adj SERA"}
    combine(FOLDER, OUTPUT, REQUIRED_COLS)
