import os
import sys
from pycharter.etl_generator import Pipeline

ROOT_DIR = os.path.join('..')
DATA_DIR = os.path.join(ROOT_DIR, 'data')

sys.path.append(ROOT_DIR)

p = Pipeline.from_config_files(
    extract='../data/etl_examples/extract.yaml',    
    transform='../data/etl_examples/transform.yaml',
    load='../data/etl_examples/load.yaml',
    variables={
        "INPUT_PATH": str(os.path.join(ROOT_DIR,"data/etl_examples/input.csv")),
        "OUTPUT_PATH": str(os.path.join(ROOT_DIR,"data/etl_examples/output2.csv")),
    }
)
r = await p.run()


r





from pycharter.runtime_validator import Validator

contract = "../data/aviation_examples/aircraft_configuration/aircraft_configuration_contract.yaml"
validator = Validator(contract_file=str(contract))
result = validator.validate({
    "metadata": {},
    "registration": "CX77G",
    "valid_since": "1996-01-01T00:00:00Z",
    "valid_until": "2013-12-31T00:00:00Z",
    "ac_operator": "CX",
    "ac_owner": "CX",
    "ac_subtype": "77G",
    "ac_logical_no": 0,
    "ac_state": "O",
    "dry_operating_wgt": 0,
    "max_takeoff_wgt": 0,
    "cargo_capacity": 0,
    "fuel_capacity": 0,
    "avg_fuel_consump": 0,
    "ac_index": 100,
    "crewsize_cockpit": 0,
    "crewsize_cabin": 0,
    "std_version": "STN",
    "ap_restriction": "N",
    "ac_owner_name": "CX",
    "ac_subtype_name": "77G",
    "ac_category": "J",
    "fuel_measure_unit": "kg",
})



# result.data
result.errors





from pymongo import MongoClient
import psycopg2
connection_string='postgresql://postgres:1234567890@ioc-de-delete-me-3-instance-1.cvpchc1tnb8b.ap-southeast-1.rds.amazonaws.com'


connect = psycopg2.connect(connection_string)


import pymongo
import pandas as pd
import json
from datetime import datetime
from bson import ObjectId
from bson.json_util import dumps

# Set up connection parameters
# NEVER hardcode credentials in production code. Use environment variables.
CONNECTION_STRING = "mongodb+srv://ioc_de_ete:RjL7xac3g2U6RULZ@ioc-de-ete.oijg8.mongodb.net/?authMechanism=SCRAM-SHA-1&tls=true" 
DB_NAME = "external"
COLLECTION_NAME = "replica_nlo_mel_to_constraints_info_serve"

# Output file paths
JSON_OUTPUT = "mel_to_constraints_info_data.json"
CSV_OUTPUT = "mel_to_constraints_info_data.csv"

try:
    # Create a MongoDB client
    client = pymongo.MongoClient(CONNECTION_STRING)

    # Access the database and collection
    database = client[DB_NAME]
    collection = database[COLLECTION_NAME]

    # Retrieve all documents (empty filter {}) using find() method
    cursor = collection.find({}).sort("created_at", -1).limit(20)

    # Convert the cursor results to a Python list of dictionaries
    list_of_documents = list(cursor)
    
    print(f"✓ Retrieved {len(list_of_documents)} documents from MongoDB")

    # ============================================================
    # Save to JSON
    # ============================================================
    # Method 1: Using bson.json_util (handles ObjectId, datetime automatically)
    with open(JSON_OUTPUT, 'w') as json_file:
        json_file.write(dumps(list_of_documents, indent=2, default=str))
    
    print(f"✓ Saved {len(list_of_documents)} documents to {JSON_OUTPUT}")
    
    # Method 2: Manual conversion (cleaner format for validation)
    def convert_mongodb_to_json(obj):
        """Convert MongoDB types to JSON-serializable types."""
        if isinstance(obj, ObjectId):
            return str(obj)
        elif isinstance(obj, datetime):
            return obj.isoformat()
        elif isinstance(obj, dict):
            return {k: convert_mongodb_to_json(v) for k, v in obj.items()}
        elif isinstance(obj, list):
            return [convert_mongodb_to_json(item) for item in obj]
        else:
            return obj
    
    json_data = convert_mongodb_to_json(list_of_documents)
    clean_json_output = JSON_OUTPUT.replace('.json', '_clean.json')
    with open(clean_json_output, 'w') as json_file:
        json.dump(json_data, json_file, indent=2, ensure_ascii=False)
    
    print(f"✓ Saved clean JSON to {clean_json_output}")

    # ============================================================
    # Save to CSV
    # ============================================================
    # Convert to pandas DataFrame
    df = pd.DataFrame(list_of_documents)
    
    # Handle MongoDB-specific types for CSV export
    # Convert ObjectId columns to strings
    for col in df.columns:
        if df[col].dtype == 'object':
            # Check if column contains ObjectId
            sample = df[col].dropna()
            if len(sample) > 0 and isinstance(sample.iloc[0], ObjectId):
                df[col] = df[col].astype(str)
            # Convert datetime to string
            elif len(sample) > 0 and isinstance(sample.iloc[0], datetime):
                df[col] = df[col].apply(lambda x: x.isoformat() if pd.notna(x) and isinstance(x, datetime) else x)
    
    # Save to CSV
    df.to_csv(CSV_OUTPUT, index=False, encoding='utf-8')
    print(f"✓ Saved {len(df)} rows to {CSV_OUTPUT}")
    print(f"  Columns: {len(df.columns)}")
    print(f"  Shape: {df.shape}")
    
    # Display preview
    print("\nData preview:")
    print(df.head())

    # Close the connection
    client.close()
    print(f"\n✓ Connection closed")
    print(f"\nFiles saved:")
    print(f"  - {JSON_OUTPUT} (MongoDB format with $oid, $date)")
    print(f"  - {clean_json_output} (Clean JSON format)")
    print(f"  - {CSV_OUTPUT}")

except Exception as e:
    print(f"An error occurred: {e}")
    import traceback
    traceback.print_exc()



df



