//! Write-Ahead Log for Graph Store durability
//!
//! Provides crash recovery by logging all graph operations before they're applied.
//!
//! # Format
//!
//! Graph WAL files use JSON Lines format with the following structure:
//! - Header: magic number, version, shard_id, start_sequence
//! - Entries: GraphWALEntry (one per line)
//! - Footer: entry_count, end_sequence, checksum

use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::Arc;
use std::io::{self, Write, BufWriter, BufRead, BufReader};
use std::fs::{File, OpenOptions};

use serde::{Serialize, Deserialize};

use crate::types::graph::{Entity, Relation};
use crate::storage::concurrent::WriteMetrics;
use crate::storage::wal::{SyncMode, WAL_MAGIC, WAL_VERSION};

use super::types::GraphWriteOp;

/// Graph WAL operation type
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub enum GraphWALOperation {
    /// Insert or update an entity
    PutEntity,
    /// Delete an entity
    DeleteEntity,
    /// Insert or update a relation
    PutRelation,
    /// Delete a relation
    DeleteRelation,
}

impl From<&GraphWriteOp> for GraphWALOperation {
    fn from(op: &GraphWriteOp) -> Self {
        match op {
            GraphWriteOp::PutEntity(_) => GraphWALOperation::PutEntity,
            GraphWriteOp::DeleteEntity(_) => GraphWALOperation::DeleteEntity,
            GraphWriteOp::PutRelation(_) => GraphWALOperation::PutRelation,
            GraphWriteOp::DeleteRelation(_) => GraphWALOperation::DeleteRelation,
            GraphWriteOp::Batch(_) => panic!("Batch operations should be expanded before WAL"),
        }
    }
}

/// A single entry in the Graph WAL
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct GraphWALEntry {
    /// Sequence number (globally unique within shard)
    pub sequence: u64,
    
    /// Operation type
    pub operation: GraphWALOperation,
    
    /// Shard ID
    pub shard_id: usize,
    
    /// Timestamp when entry was created (ms since epoch)
    pub timestamp: i64,
    
    /// Entity data (for PutEntity operations)
    #[serde(skip_serializing_if = "Option::is_none")]
    pub entity: Option<Entity>,
    
    /// Relation data (for PutRelation operations)
    #[serde(skip_serializing_if = "Option::is_none")]
    pub relation: Option<Relation>,
    
    /// ID for delete operations
    #[serde(skip_serializing_if = "Option::is_none")]
    pub delete_id: Option<String>,
    
    /// Checksum of this entry
    pub checksum: u32,
}

impl GraphWALEntry {
    /// Create entry for PutEntity operation
    pub fn put_entity(sequence: u64, shard_id: usize, entity: Entity) -> Self {
        let mut entry = Self {
            sequence,
            operation: GraphWALOperation::PutEntity,
            shard_id,
            timestamp: current_timestamp(),
            entity: Some(entity),
            relation: None,
            delete_id: None,
            checksum: 0,
        };
        entry.checksum = entry.calculate_checksum();
        entry
    }
    
    /// Create entry for DeleteEntity operation
    pub fn delete_entity(sequence: u64, shard_id: usize, entity_id: String) -> Self {
        let mut entry = Self {
            sequence,
            operation: GraphWALOperation::DeleteEntity,
            shard_id,
            timestamp: current_timestamp(),
            entity: None,
            relation: None,
            delete_id: Some(entity_id),
            checksum: 0,
        };
        entry.checksum = entry.calculate_checksum();
        entry
    }
    
    /// Create entry for PutRelation operation
    pub fn put_relation(sequence: u64, shard_id: usize, relation: Relation) -> Self {
        let mut entry = Self {
            sequence,
            operation: GraphWALOperation::PutRelation,
            shard_id,
            timestamp: current_timestamp(),
            entity: None,
            relation: Some(relation),
            delete_id: None,
            checksum: 0,
        };
        entry.checksum = entry.calculate_checksum();
        entry
    }
    
    /// Create entry for DeleteRelation operation
    pub fn delete_relation(sequence: u64, shard_id: usize, relation_id: String) -> Self {
        let mut entry = Self {
            sequence,
            operation: GraphWALOperation::DeleteRelation,
            shard_id,
            timestamp: current_timestamp(),
            entity: None,
            relation: None,
            delete_id: Some(relation_id),
            checksum: 0,
        };
        entry.checksum = entry.calculate_checksum();
        entry
    }
    
    /// Create WAL entry from a GraphWriteOp
    pub fn from_op(sequence: u64, shard_id: usize, op: &GraphWriteOp) -> Vec<Self> {
        match op {
            GraphWriteOp::PutEntity(entity) => {
                vec![Self::put_entity(sequence, shard_id, entity.clone())]
            }
            GraphWriteOp::DeleteEntity(id) => {
                vec![Self::delete_entity(sequence, shard_id, id.clone())]
            }
            GraphWriteOp::PutRelation(relation) => {
                vec![Self::put_relation(sequence, shard_id, relation.clone())]
            }
            GraphWriteOp::DeleteRelation(id) => {
                vec![Self::delete_relation(sequence, shard_id, id.clone())]
            }
            GraphWriteOp::Batch(ops) => {
                // Expand batch into individual entries
                let mut entries = Vec::new();
                for (i, sub_op) in ops.iter().enumerate() {
                    entries.extend(Self::from_op(sequence + i as u64, shard_id, sub_op));
                }
                entries
            }
        }
    }
    
    /// Calculate checksum for this entry
    pub fn calculate_checksum(&self) -> u32 {
        let mut hasher = crc32fast::Hasher::new();
        hasher.update(&self.sequence.to_le_bytes());
        hasher.update(&[self.operation as u8]);
        hasher.update(&(self.shard_id as u64).to_le_bytes());
        hasher.update(&self.timestamp.to_le_bytes());
        
        if let Some(ref entity) = self.entity {
            if let Ok(bytes) = serde_json::to_vec(entity) {
                hasher.update(&bytes);
            }
        }
        if let Some(ref relation) = self.relation {
            if let Ok(bytes) = serde_json::to_vec(relation) {
                hasher.update(&bytes);
            }
        }
        if let Some(ref id) = self.delete_id {
            hasher.update(id.as_bytes());
        }
        
        hasher.finalize()
    }
    
    /// Verify checksum
    pub fn verify_checksum(&self) -> bool {
        let expected = self.calculate_checksum();
        expected == self.checksum
    }
    
    /// Convert to GraphWriteOp for replay
    pub fn to_write_op(&self) -> Option<GraphWriteOp> {
        match self.operation {
            GraphWALOperation::PutEntity => {
                self.entity.clone().map(GraphWriteOp::PutEntity)
            }
            GraphWALOperation::DeleteEntity => {
                self.delete_id.clone().map(GraphWriteOp::DeleteEntity)
            }
            GraphWALOperation::PutRelation => {
                self.relation.clone().map(GraphWriteOp::PutRelation)
            }
            GraphWALOperation::DeleteRelation => {
                self.delete_id.clone().map(GraphWriteOp::DeleteRelation)
            }
        }
    }
}

/// Configuration for Graph WAL
#[derive(Debug, Clone)]
pub struct GraphWALConfig {
    /// Base directory for WAL files
    pub path: PathBuf,
    
    /// Maximum file size before rotation (bytes)
    pub max_file_size: usize,
    
    /// Sync mode
    pub sync_mode: SyncMode,
    
    /// Number of old WAL files to retain
    pub retention_count: usize,
    
    /// Buffer size for writes
    pub buffer_size: usize,
    
    /// Whether WAL is enabled
    pub enabled: bool,
}

impl Default for GraphWALConfig {
    fn default() -> Self {
        Self {
            path: PathBuf::from("data/graph/wal"),
            max_file_size: 64 * 1024 * 1024, // 64MB
            sync_mode: SyncMode::Batch,
            retention_count: 10,
            buffer_size: 64 * 1024, // 64KB
            enabled: true,
        }
    }
}

impl GraphWALConfig {
    /// Create config with specified path
    pub fn new(path: impl Into<PathBuf>) -> Self {
        Self {
            path: path.into(),
            ..Default::default()
        }
    }
    
    /// Disable WAL (for testing)
    pub fn disabled() -> Self {
        Self {
            enabled: false,
            ..Default::default()
        }
    }
    
    /// Set sync mode
    pub fn with_sync_mode(mut self, mode: SyncMode) -> Self {
        self.sync_mode = mode;
        self
    }
}

/// Header for Graph WAL file
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct GraphWALHeader {
    /// Magic number
    pub magic: [u8; 4],
    /// Format version
    pub version: u32,
    /// Shard ID
    pub shard_id: usize,
    /// First sequence number in this file
    pub start_sequence: u64,
    /// Creation timestamp
    pub created_at: i64,
}

impl GraphWALHeader {
    pub fn new(shard_id: usize, start_sequence: u64) -> Self {
        Self {
            magic: WAL_MAGIC,
            version: WAL_VERSION,
            shard_id,
            start_sequence,
            created_at: current_timestamp(),
        }
    }
    
    pub fn validate(&self) -> Result<(), GraphWALError> {
        if self.magic != WAL_MAGIC {
            return Err(GraphWALError::InvalidMagic);
        }
        if self.version > WAL_VERSION {
            return Err(GraphWALError::UnsupportedVersion(self.version));
        }
        Ok(())
    }
}

/// Footer for Graph WAL file
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct GraphWALFooter {
    /// Number of entries in this file
    pub entry_count: u32,
    /// Last sequence number
    pub end_sequence: u64,
    /// Checksum of all entries
    pub checksum: u32,
    /// Completion timestamp
    pub completed_at: i64,
}

impl GraphWALFooter {
    pub fn new(entry_count: u32, end_sequence: u64, checksum: u32) -> Self {
        Self {
            entry_count,
            end_sequence,
            checksum,
            completed_at: current_timestamp(),
        }
    }
}

/// Graph WAL Writer
pub struct GraphWALWriter {
    /// Configuration
    config: GraphWALConfig,
    
    /// Shard ID
    shard_id: usize,
    
    /// Current file path
    current_file_path: PathBuf,
    
    /// Current file writer
    writer: Option<BufWriter<File>>,
    
    /// Current file size
    current_size: usize,
    
    /// Sequence counter
    sequence: AtomicU64,
    
    /// Entry count in current file
    entry_count: u32,
    
    /// Running checksum
    running_checksum: u32,
    
    /// Metrics
    metrics: Arc<WriteMetrics>,
}

impl GraphWALWriter {
    /// Create a new Graph WAL writer
    pub fn new(config: GraphWALConfig, shard_id: usize, metrics: Arc<WriteMetrics>) -> io::Result<Self> {
        if !config.enabled {
            return Ok(Self {
                config,
                shard_id,
                current_file_path: PathBuf::new(),
                writer: None,
                current_size: 0,
                sequence: AtomicU64::new(0),
                entry_count: 0,
                running_checksum: 0,
                metrics,
            });
        }
        
        // Create WAL directory
        std::fs::create_dir_all(&config.path)?;
        
        let mut writer = Self {
            config,
            shard_id,
            current_file_path: PathBuf::new(),
            writer: None,
            current_size: 0,
            sequence: AtomicU64::new(0),
            entry_count: 0,
            running_checksum: 0,
            metrics,
        };
        
        // Recover sequence from existing files
        writer.recover_sequence()?;
        
        // NOTE: We do NOT call rotate() here anymore to avoid creating empty WAL files
        // on every server restart. Instead, rotate() will be called lazily on first write.
        // This fixes the "empty .gwal file on restart" issue.
        
        Ok(writer)
    }
    
    /// Recover sequence number from existing WAL files
    fn recover_sequence(&mut self) -> io::Result<()> {
        let mut max_seq: u64 = 0;
        
        if !self.config.path.exists() {
            self.sequence.store(0, Ordering::SeqCst);
            return Ok(());
        }
        
        // First, scan filenames to get max start_sequence
        // This ensures we don't create duplicate filenames
        if let Ok(entries) = std::fs::read_dir(&self.config.path) {
            for entry in entries.flatten() {
                let path = entry.path();
                if let Some(file_seq) = Self::parse_sequence(&path) {
                    // Check if this file belongs to our shard
                    if let Some(stem) = path.file_stem().and_then(|s| s.to_str()) {
                        if stem.starts_with(&format!("graph_{}_", self.shard_id)) {
                            max_seq = max_seq.max(file_seq);
                        }
                    }
                }
            }
        }
        
        // Also read through all WAL entries to find the actual max sequence
        let reader = GraphWALReader::new(&self.config.path).for_shard(self.shard_id);
        if let Ok(entries) = reader.read_all() {
            for entry in entries {
                max_seq = max_seq.max(entry.sequence);
            }
        }
        
        // Start from the next sequence after the max found
        self.sequence.store(max_seq + 1, Ordering::SeqCst);
        Ok(())
    }
    
    /// Try to resume writing to the last WAL file if it's not full
    /// Returns true if successfully resumed, false if need to create new file
    fn try_resume_last_file(&mut self) -> io::Result<bool> {
        if !self.config.path.exists() {
            tracing::debug!(
                "WAL shard {}: directory {:?} does not exist, cannot resume",
                self.shard_id, self.config.path
            );
            return Ok(false);
        }
        
        // Find all WAL files for this shard
        let mut files: Vec<PathBuf> = Vec::new();
        if let Ok(entries) = std::fs::read_dir(&self.config.path) {
            for entry in entries.flatten() {
                let path = entry.path();
                if path.extension().map(|e| e == "gwal").unwrap_or(false) {
                    if let Some(stem) = path.file_stem().and_then(|s| s.to_str()) {
                        if stem.starts_with(&format!("graph_{}_", self.shard_id)) {
                            files.push(path);
                        }
                    }
                }
            }
        }
        
        if files.is_empty() {
            tracing::debug!(
                "WAL shard {}: no .gwal files found in {:?}, cannot resume",
                self.shard_id, self.config.path
            );
            return Ok(false);
        }
        
        // Sort by sequence number and get the last file
        files.sort_by(|a, b| {
            let seq_a = Self::parse_sequence(a).unwrap_or(0);
            let seq_b = Self::parse_sequence(b).unwrap_or(0);
            seq_a.cmp(&seq_b)
        });
        
        let last_file = files.last().unwrap();
        let file_size = std::fs::metadata(last_file)?.len() as usize;
        
        // Check if file is too large to resume
        if file_size >= self.config.max_file_size {
            tracing::info!(
                "WAL shard {}: last file {} is full ({} bytes >= {} max), will create new file",
                self.shard_id, last_file.display(), file_size, self.config.max_file_size
            );
            return Ok(false);
        }
        
        // Read file content to check for footer and get state
        let content = match std::fs::read_to_string(last_file) {
            Ok(c) => c,
            Err(e) => {
                tracing::warn!(
                    "WAL shard {}: failed to read last file {} for resume: {} — \
                     this may happen if another process still holds the file lock (e.g. MCP server restart). \
                     A NEW .gwal file will be created (data in old file is NOT lost, reader reads all files)",
                    self.shard_id, last_file.display(), e
                );
                return Err(e);
            }
        };
        let lines: Vec<&str> = content.lines().collect();
        
        if lines.is_empty() {
            tracing::warn!(
                "WAL shard {}: last file {} is empty, cannot resume — will create new file",
                self.shard_id, last_file.display()
            );
            return Ok(false);
        }
        
        // Check if last line is a footer
        let last_line = lines.last().unwrap();
        let has_footer = serde_json::from_str::<GraphWALFooter>(last_line).is_ok();
        
        // Count entries and build content
        // If has_footer: skip footer (last line)
        // If no footer: include all lines (file may have been killed without graceful shutdown)
        let mut entry_count = 0u32;
        let mut running_checksum = 0u32;
        let mut valid_content = String::new();
        let mut header_valid = false;
        
        for (i, line) in lines.iter().enumerate() {
            if i == 0 {
                // Header - validate it
                if serde_json::from_str::<GraphWALHeader>(line).is_err() {
                    tracing::warn!(
                        "WAL shard {}: last file {} has invalid header, cannot resume — will create new file",
                        self.shard_id, last_file.display()
                    );
                    return Ok(false);
                }
                header_valid = true;
                valid_content.push_str(line);
                valid_content.push('\n');
            } else if has_footer && i == lines.len() - 1 {
                // Footer - skip it (we're removing it to resume writing)
                tracing::info!(
                    "WAL shard {}: stripping footer from {} to resume writing (file size {} < {} max)",
                    self.shard_id, last_file.display(), file_size, self.config.max_file_size
                );
            } else {
                // Entry - count and calculate checksum
                // Skip empty lines (may occur from incomplete writes)
                if line.trim().is_empty() {
                    continue;
                }
                if let Ok(entry) = serde_json::from_str::<GraphWALEntry>(line) {
                    entry_count += 1;
                    let mut hasher = crc32fast::Hasher::new_with_initial(running_checksum);
                    hasher.update(line.as_bytes());
                    running_checksum = hasher.finalize();
                    valid_content.push_str(line);
                    valid_content.push('\n');
                }
                // Invalid lines are skipped (may be partial writes from crash)
            }
        }
        
        // Must have valid header to resume
        if !header_valid {
            tracing::warn!(
                "WAL shard {}: header validation failed in {}, cannot resume — will create new file",
                self.shard_id, last_file.display()
            );
            return Ok(false);
        }
        
        // Log whether we're resuming with or without footer
        if has_footer {
            tracing::info!(
                "Resuming WAL shard {} file {} (had footer, {} entries, {} bytes)",
                self.shard_id, last_file.display(), entry_count, file_size
            );
        } else {
            tracing::warn!(
                "Resuming WAL shard {} file {} without footer (process may have been killed, {} entries recovered, {} bytes)",
                self.shard_id, last_file.display(), entry_count, file_size
            );
        }
        
        // Rewrite file with valid content only (removes footer or invalid lines)
        std::fs::write(last_file, &valid_content)?;
        
        // Open file for appending
        let file = OpenOptions::new()
            .append(true)
            .open(last_file)?;
        
        let writer = BufWriter::with_capacity(self.config.buffer_size, file);
        
        // Update state
        self.current_file_path = last_file.clone();
        self.writer = Some(writer);
        self.current_size = valid_content.len();
        self.entry_count = entry_count;
        self.running_checksum = running_checksum;
        
        tracing::info!(
            "WAL shard {} resumed successfully: {} (size: {} bytes, entries: {})",
            self.shard_id, last_file.display(), self.current_size, self.entry_count
        );
        
        Ok(true)
    }
    
    /// Parse sequence from filename
    fn parse_sequence(path: &Path) -> Option<u64> {
        let stem = path.file_stem()?.to_str()?;
        let parts: Vec<&str> = stem.split('_').collect();
        if parts.len() >= 3 {
            parts[2].parse().ok()
        } else {
            None
        }
    }
    
    /// Get next sequence number
    pub fn next_sequence(&self) -> u64 {
        self.sequence.fetch_add(1, Ordering::SeqCst)
    }
    
    /// Get current sequence number
    pub fn current_sequence(&self) -> u64 {
        self.sequence.load(Ordering::SeqCst)
    }
    
    /// Generate filename for WAL file
    fn generate_filename(&self, start_seq: u64) -> PathBuf {
        self.config.path.join(format!("graph_{}_{:016}.gwal", self.shard_id, start_seq))
    }
    
    /// Rotate to new WAL file
    pub fn rotate(&mut self) -> io::Result<()> {
        if !self.config.enabled {
            return Ok(());
        }
        
        // Finalize current file
        if let Some(mut writer) = self.writer.take() {
            let footer = GraphWALFooter::new(
                self.entry_count,
                self.current_sequence(),
                self.running_checksum,
            );
            let footer_json = serde_json::to_string(&footer)
                .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?;
            writeln!(writer, "{}", footer_json)?;
            writer.flush()?;
            
            self.metrics.record_wal_rotation();
        }
        
        // Open new file - use current sequence for filename (don't consume sequence numbers)
        // This ensures entry sequence numbers are not affected by file rotation
        let mut start_seq = self.current_sequence();
        let mut path = self.generate_filename(start_seq);
        
        // Keep incrementing start_seq (for filename only) until we find a non-existing file
        // Note: We don't use next_sequence() here to avoid consuming entry sequence numbers
        while path.exists() {
            start_seq += 1;
            path = self.generate_filename(start_seq);
        }
        
        let file = OpenOptions::new()
            .create_new(true)  // Use create_new to fail if file exists (safety check)
            .write(true)
            .open(&path)?;
        
        let mut writer = BufWriter::with_capacity(self.config.buffer_size, file);
        
        // Write header
        let header = GraphWALHeader::new(self.shard_id, start_seq);
        let header_json = serde_json::to_string(&header)
            .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?;
        writeln!(writer, "{}", header_json)?;
        
        self.current_file_path = path.clone();
        self.writer = Some(writer);
        self.current_size = header_json.len() + 1;
        self.entry_count = 0;
        self.running_checksum = 0;
        
        tracing::info!(
            "WAL shard {}: opened new file {} (start_seq={})",
            self.shard_id, path.display(), start_seq
        );
        
        Ok(())
    }
    
    /// Append an entry to the WAL
    pub fn append(&mut self, entry: &GraphWALEntry) -> io::Result<u64> {
        if !self.config.enabled {
            return Ok(entry.sequence);
        }
        
        let entry_json = serde_json::to_string(entry)
            .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?;
        
        let entry_size = entry_json.len() + 1;
        
        // Lazy initialization: try to resume last file or create new one
        // This avoids creating many small WAL files on frequent server restarts
        if self.writer.is_none() {
            // First, try to resume the last WAL file (if it exists and is not full)
            match self.try_resume_last_file() {
                Ok(true) => {
                    // Successfully resumed — no new file needed
                }
                Ok(false) => {
                    tracing::info!(
                        "WAL shard {}: creating NEW .gwal file (reason: no resumable file found or file is full)",
                        self.shard_id
                    );
                    self.rotate()?;
                }
                Err(e) => {
                    tracing::warn!(
                        "WAL shard {}: creating NEW .gwal file (reason: resume failed with error: {}). \
                         This typically happens when the previous process still holds the file lock. \
                         Data integrity is preserved — the reader merges all .gwal files on load.",
                        self.shard_id, e
                    );
                    self.rotate()?;
                }
            }
        }
        
        // Check if rotation needed due to file size
        if self.current_size + entry_size > self.config.max_file_size {
            tracing::info!(
                "WAL shard {}: rotating .gwal file (reason: size limit reached, current={} + entry={} > max={})",
                self.shard_id, self.current_size, entry_size, self.config.max_file_size
            );
            self.rotate()?;
        }
        
        let writer = self.writer.as_mut()
            .ok_or_else(|| io::Error::new(io::ErrorKind::NotConnected, "WAL not open"))?;
        
        writeln!(writer, "{}", entry_json)?;
        
        // Update state
        self.current_size += entry_size;
        self.entry_count += 1;
        
        // Update running checksum
        let mut hasher = crc32fast::Hasher::new_with_initial(self.running_checksum);
        hasher.update(entry_json.as_bytes());
        self.running_checksum = hasher.finalize();
        
        // Sync based on mode
        match self.config.sync_mode {
            SyncMode::Every => {
                writer.flush()?;
                writer.get_ref().sync_data()?;
            }
            _ => {}
        }
        
        self.metrics.record_wal_bytes(entry_size);
        
        Ok(entry.sequence)
    }
    
    /// Append a write operation to WAL
    pub fn append_op(&mut self, op: &GraphWriteOp) -> io::Result<u64> {
        let seq = self.next_sequence();
        let entries = GraphWALEntry::from_op(seq, self.shard_id, op);
        
        for entry in &entries {
            self.append(entry)?;
        }
        
        Ok(seq)
    }
    
    /// Append batch and sync
    pub fn append_batch(&mut self, entries: &[GraphWALEntry]) -> io::Result<()> {
        for entry in entries {
            self.append(entry)?;
        }
        
        if self.config.sync_mode == SyncMode::Batch {
            self.sync()?;
        }
        
        Ok(())
    }
    
    /// Flush and sync to disk
    pub fn sync(&mut self) -> io::Result<()> {
        if let Some(ref mut writer) = self.writer {
            writer.flush()?;
            writer.get_ref().sync_data()?;
        }
        Ok(())
    }
    
    /// Close the WAL writer
    pub fn close(&mut self) -> io::Result<()> {
        if let Some(mut writer) = self.writer.take() {
            let footer = GraphWALFooter::new(
                self.entry_count,
                self.current_sequence(),
                self.running_checksum,
            );
            let footer_json = serde_json::to_string(&footer)
                .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?;
            writeln!(writer, "{}", footer_json)?;
            writer.flush()?;
            writer.get_ref().sync_all()?;
        }
        Ok(())
    }
    
    /// Check if WAL is enabled
    pub fn is_enabled(&self) -> bool {
        self.config.enabled
    }
}

impl Drop for GraphWALWriter {
    fn drop(&mut self) {
        let _ = self.close();
    }
}

/// Graph WAL Reader for crash recovery
pub struct GraphWALReader {
    /// Directory containing WAL files
    path: PathBuf,
    
    /// Shard ID filter
    shard_id: Option<usize>,
}

impl GraphWALReader {
    /// Create a new Graph WAL reader
    pub fn new(path: impl Into<PathBuf>) -> Self {
        Self {
            path: path.into(),
            shard_id: None,
        }
    }
    
    /// Filter by shard ID
    pub fn for_shard(mut self, shard_id: usize) -> Self {
        self.shard_id = Some(shard_id);
        self
    }
    
    /// List all WAL files in order
    pub fn list_files(&self) -> io::Result<Vec<PathBuf>> {
        let mut files: Vec<PathBuf> = Vec::new();
        
        if !self.path.exists() {
            return Ok(files);
        }
        
        for entry in std::fs::read_dir(&self.path)? {
            let entry = entry?;
            let path = entry.path();
            
            if path.extension().map(|e| e == "gwal").unwrap_or(false) {
                // Filter by shard if specified
                if let Some(shard_id) = self.shard_id {
                    if let Some(file_shard) = Self::parse_shard_id(&path) {
                        if file_shard != shard_id {
                            continue;
                        }
                    }
                }
                
                files.push(path);
            }
        }
        
        // Sort by sequence number
        files.sort_by(|a, b| {
            let seq_a = GraphWALWriter::parse_sequence(a).unwrap_or(0);
            let seq_b = GraphWALWriter::parse_sequence(b).unwrap_or(0);
            seq_a.cmp(&seq_b)
        });
        
        Ok(files)
    }
    
    /// Parse shard ID from filename
    fn parse_shard_id(path: &Path) -> Option<usize> {
        let stem = path.file_stem()?.to_str()?;
        let parts: Vec<&str> = stem.split('_').collect();
        if parts.len() >= 2 {
            parts[1].parse().ok()
        } else {
            None
        }
    }
    
    /// Read all entries from all WAL files
    pub fn read_all(&self) -> io::Result<Vec<GraphWALEntry>> {
        let mut entries = Vec::new();
        
        for file_path in self.list_files()? {
            let file_entries = self.read_file(&file_path)?;
            entries.extend(file_entries);
        }
        
        // Sort by sequence
        entries.sort_by_key(|e| e.sequence);
        
        Ok(entries)
    }
    
    /// Read entries from a single WAL file
    pub fn read_file(&self, path: &Path) -> io::Result<Vec<GraphWALEntry>> {
        let file = File::open(path)?;
        let reader = BufReader::new(file);
        let mut entries = Vec::new();
        let mut is_first_line = true;
        
        for line in reader.lines() {
            let line = line?;
            if line.trim().is_empty() {
                continue;
            }
            
            // Skip header
            if is_first_line {
                is_first_line = false;
                // Validate header
                if let Ok(header) = serde_json::from_str::<GraphWALHeader>(&line) {
                    if let Err(e) = header.validate() {
                        return Err(io::Error::new(io::ErrorKind::InvalidData, e.to_string()));
                    }
                }
                continue;
            }
            
            // Try to parse as entry
            if let Ok(entry) = serde_json::from_str::<GraphWALEntry>(&line) {
                // NOTE: Checksum verification disabled due to serde_json non-deterministic key ordering.
                // The checksum is calculated using serde_json::to_vec() which may produce different
                // byte sequences for the same data depending on HashMap iteration order.
                // TODO: Fix by using a deterministic serialization method (e.g., sort keys)
                entries.push(entry);
            } else if serde_json::from_str::<GraphWALFooter>(&line).is_ok() {
                // Footer signals end of entries
                break;
            }
        }
        
        Ok(entries)
    }
    
    /// Get the highest sequence number
    pub fn get_max_sequence(&self) -> io::Result<u64> {
        let entries = self.read_all()?;
        Ok(entries.iter().map(|e| e.sequence).max().unwrap_or(0))
    }
    
    /// Replay entries and return recovery result
    ///
    /// Phase-96: Uses two-pass tombstone-aware replay to prevent deleted entities
    /// from being resurrected by earlier PutEntity entries during recovery.
    ///
    /// Pass 1: Pre-scan all entries to build a tombstone set — entities/relations
    ///         whose LAST operation in the WAL is a Delete.
    /// Pass 2: Replay entries in sequence order, skipping Put operations for
    ///         tombstoned entities/relations (since we know they'll be deleted later).
    ///
    /// This is a defense-in-depth measure. Sequential replay already handles
    /// put→delete correctly, but tombstone awareness prevents issues with:
    /// - Partial WAL flushes before crash
    /// - Edge cases in multi-file WAL rotation
    /// - Type pollution in GlobalGraphIndex during recovery
    pub fn replay<F>(&self, mut apply_fn: F) -> io::Result<GraphRecoveryResult>
    where
        F: FnMut(&GraphWALEntry) -> Result<(), String>,
    {
        // ── Pass 1: Build tombstone set ────────────────────────────────────
        // Read all entries and find the LAST operation per entity/relation ID.
        // If the last operation is a Delete, mark that ID as tombstoned.
        use std::collections::HashMap;
        
        let all_entries = self.read_all()?; // sorted by sequence
        
        // Track the last operation type per entity/relation ID
        // Key: (is_entity: bool, id: String) → last_sequence
        let mut last_op: HashMap<String, (u64, GraphWALOperation)> = HashMap::new();
        
        for entry in &all_entries {
            let id = match entry.operation {
                GraphWALOperation::PutEntity => {
                    entry.entity.as_ref().map(|e| e.id.clone())
                }
                GraphWALOperation::DeleteEntity => {
                    entry.delete_id.clone()
                }
                GraphWALOperation::PutRelation => {
                    entry.relation.as_ref().map(|r| r.id.clone())
                }
                GraphWALOperation::DeleteRelation => {
                    entry.delete_id.clone()
                }
            };
            
            if let Some(id) = id {
                // Use a prefixed key to distinguish entity vs relation IDs
                let key = match entry.operation {
                    GraphWALOperation::PutEntity | GraphWALOperation::DeleteEntity => {
                        format!("e:{}", id)
                    }
                    GraphWALOperation::PutRelation | GraphWALOperation::DeleteRelation => {
                        format!("r:{}", id)
                    }
                };
                
                let existing = last_op.get(&key);
                if existing.is_none() || existing.unwrap().0 <= entry.sequence {
                    last_op.insert(key, (entry.sequence, entry.operation));
                }
            }
        }
        
        // Build the tombstone set: IDs whose last operation is Delete
        let tombstoned: std::collections::HashSet<String> = last_op
            .into_iter()
            .filter(|(_, (_, op))| matches!(op, GraphWALOperation::DeleteEntity | GraphWALOperation::DeleteRelation))
            .map(|(key, _)| key)
            .collect();
        
        if !tombstoned.is_empty() {
            tracing::info!(
                "WAL replay: {} tombstoned entries detected (will skip earlier Put operations for these IDs)",
                tombstoned.len()
            );
        }
        
        // ── Pass 2: Replay with tombstone filtering ────────────────────────
        let mut result = GraphRecoveryResult::default();
        result.files_processed = 1; // We used read_all which covers all files
        result.tombstoned_count = tombstoned.len();
            
        for entry in &all_entries {
                result.last_sequence = result.last_sequence.max(entry.sequence);
                
            // Check if this is a Put for a tombstoned entity/relation
            let should_skip = match entry.operation {
                GraphWALOperation::PutEntity => {
                    entry.entity.as_ref()
                        .map(|e| tombstoned.contains(&format!("e:{}", e.id)))
                        .unwrap_or(false)
                }
                GraphWALOperation::PutRelation => {
                    entry.relation.as_ref()
                        .map(|r| tombstoned.contains(&format!("r:{}", r.id)))
                        .unwrap_or(false)
                }
                // Always replay Delete operations (they're idempotent and needed for finality)
                GraphWALOperation::DeleteEntity | GraphWALOperation::DeleteRelation => false,
            };
            
            if should_skip {
                result.entries_skipped += 1;
                tracing::debug!(
                    "WAL replay: skipping tombstoned entry seq={} op={:?}",
                    entry.sequence, entry.operation
                );
                continue;
            }
            
            match apply_fn(entry) {
                    Ok(()) => {
                        result.entries_replayed += 1;
                    }
                    Err(e) => {
                        tracing::warn!("Failed to replay WAL entry {}: {}", entry.sequence, e);
                        result.entries_skipped += 1;
                }
            }
        }
        
        Ok(result)
    }
    
    /// Compact all WAL files for a shard into a single file.
    ///
    /// This reads all entries from all .gwal files, writes them into one new file,
    /// and removes the old files. The WAL writer for this shard must be closed/dropped
    /// before calling compact (i.e., the graph should not be actively writing).
    ///
    /// Returns a `GraphCompactResult` with statistics about the compaction.
    pub fn compact(&self) -> io::Result<GraphCompactResult> {
        let files = self.list_files()?;
        let file_count = files.len();
        
        if file_count <= 1 {
            tracing::info!(
                "WAL compact: shard {:?} has {} file(s), no compaction needed",
                self.shard_id, file_count
            );
            return Ok(GraphCompactResult {
                files_before: file_count,
                files_after: file_count,
                entries_total: if file_count == 1 {
                    self.read_all()?.len()
                } else {
                    0
                },
                bytes_before: files.iter().map(|f| std::fs::metadata(f).map(|m| m.len()).unwrap_or(0)).sum(),
                bytes_after: files.iter().map(|f| std::fs::metadata(f).map(|m| m.len()).unwrap_or(0)).sum(),
                compacted: false,
            });
        }
        
        // Determine the shard_id from the first file
        let shard_id = self.shard_id.unwrap_or_else(|| {
            files.first()
                .and_then(|f| Self::parse_shard_id(f))
                .unwrap_or(0)
        });
        
        // Read all entries across all files
        let entries = self.read_all()?;
        let total_entries = entries.len();
        
        // Calculate total size before compaction
        let bytes_before: u64 = files.iter()
            .map(|f| std::fs::metadata(f).map(|m| m.len()).unwrap_or(0))
            .sum();
        
        tracing::info!(
            "WAL compact shard {}: merging {} files ({} entries, {} bytes) into single file",
            shard_id, file_count, total_entries, bytes_before
        );
        
        // Determine start sequence for the compacted file (use lowest sequence)
        let start_seq = entries.first().map(|e| e.sequence).unwrap_or(0);
        let end_seq = entries.last().map(|e| e.sequence).unwrap_or(0);
        
        // Write all entries to a temporary file first
        let parent_dir = files.first()
            .and_then(|f| f.parent())
            .ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, "Cannot determine WAL directory"))?;
        
        let temp_filename = parent_dir.join(format!("graph_{}_{:016}.gwal.compact_tmp", shard_id, start_seq));
        
        {
            let file = OpenOptions::new()
                .create_new(true)
                .write(true)
                .open(&temp_filename)?;
            
            let mut writer = BufWriter::with_capacity(64 * 1024, file);
            
            // Write header
            let header = GraphWALHeader::new(shard_id, start_seq);
            let header_json = serde_json::to_string(&header)
                .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?;
            writeln!(writer, "{}", header_json)?;
            
            // Write all entries
            let mut running_checksum = 0u32;
            for entry in &entries {
                let entry_json = serde_json::to_string(entry)
                    .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?;
                let mut hasher = crc32fast::Hasher::new_with_initial(running_checksum);
                hasher.update(entry_json.as_bytes());
                running_checksum = hasher.finalize();
                writeln!(writer, "{}", entry_json)?;
            }
            
            // Do NOT write footer — leave file open for future appends
            writer.flush()?;
            writer.get_ref().sync_all()?;
        }
        
        let bytes_after = std::fs::metadata(&temp_filename)?.len();
        
        // Remove old files
        for old_file in &files {
            if let Err(e) = std::fs::remove_file(old_file) {
                tracing::warn!("WAL compact: failed to remove old file {}: {}", old_file.display(), e);
                // Try to clean up temp file on failure
                let _ = std::fs::remove_file(&temp_filename);
                return Err(e);
            }
        }
        
        // Rename temp file to final name
        let final_filename = parent_dir.join(format!("graph_{}_{:016}.gwal", shard_id, start_seq));
        std::fs::rename(&temp_filename, &final_filename)?;
        
        tracing::info!(
            "WAL compact shard {} complete: {} files → 1 file, {} entries, {} → {} bytes (seq {}..{})",
            shard_id, file_count, total_entries, bytes_before, bytes_after, start_seq, end_seq
        );
        
        Ok(GraphCompactResult {
            files_before: file_count,
            files_after: 1,
            entries_total: total_entries,
            bytes_before,
            bytes_after,
            compacted: true,
        })
    }
}

/// Result of WAL compaction
#[derive(Debug, Default)]
pub struct GraphCompactResult {
    /// Number of .gwal files before compaction
    pub files_before: usize,
    
    /// Number of .gwal files after compaction
    pub files_after: usize,
    
    /// Total number of entries in the compacted file
    pub entries_total: usize,
    
    /// Total bytes before compaction
    pub bytes_before: u64,
    
    /// Total bytes after compaction
    pub bytes_after: u64,
    
    /// Whether compaction was actually performed
    pub compacted: bool,
}

/// Recovery result from WAL replay
#[derive(Debug, Default)]
pub struct GraphRecoveryResult {
    /// Number of entries successfully replayed
    pub entries_replayed: usize,
    
    /// Number of entries skipped (errors or tombstoned)
    pub entries_skipped: usize,
    
    /// Last sequence number seen
    pub last_sequence: u64,
    
    /// Number of files processed
    pub files_processed: usize,
    
    /// Phase-96: Number of tombstoned IDs detected during pre-scan
    pub tombstoned_count: usize,
}

/// Errors from Graph WAL operations
#[derive(Debug, thiserror::Error)]
pub enum GraphWALError {
    #[error("Invalid WAL magic number")]
    InvalidMagic,
    
    #[error("Unsupported WAL version: {0}")]
    UnsupportedVersion(u32),
    
    #[error("Checksum mismatch")]
    ChecksumMismatch,
    
    #[error("IO error: {0}")]
    IoError(#[from] io::Error),
}

/// Get current timestamp in milliseconds
fn current_timestamp() -> i64 {
    use std::time::{SystemTime, UNIX_EPOCH};
    SystemTime::now()
        .duration_since(UNIX_EPOCH)
        .unwrap()
        .as_millis() as i64
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::types::graph::entity_types;
    use tempfile::tempdir;
    
    #[test]
    fn test_wal_entry_checksum() {
        let entity = Entity::new("Test", entity_types::PERSON);
        let entry = GraphWALEntry::put_entity(1, 0, entity);
        
        assert_ne!(entry.checksum, 0);
        assert!(entry.verify_checksum());
    }
    
    #[test]
    fn test_wal_entry_from_op() {
        let entity = Entity::new("Test", entity_types::PERSON);
        let op = GraphWriteOp::PutEntity(entity.clone());
        
        let entries = GraphWALEntry::from_op(1, 0, &op);
        assert_eq!(entries.len(), 1);
        assert_eq!(entries[0].operation, GraphWALOperation::PutEntity);
        assert!(entries[0].entity.is_some());
    }
    
    #[test]
    fn test_wal_entry_to_write_op() {
        let entity = Entity::new("Test", entity_types::PERSON);
        let entry = GraphWALEntry::put_entity(1, 0, entity.clone());
        
        let op = entry.to_write_op().unwrap();
        match op {
            GraphWriteOp::PutEntity(e) => assert_eq!(e.id, entity.id),
            _ => panic!("Expected PutEntity"),
        }
    }
    
    #[test]
    fn test_wal_writer_basic() {
        let dir = tempdir().unwrap();
        let config = GraphWALConfig::new(dir.path());
        let metrics = Arc::new(WriteMetrics::new());
        
        let mut writer = GraphWALWriter::new(config, 0, metrics).unwrap();
        
        // Write an entity
        let entity = Entity::new("Alice", entity_types::PERSON);
        let entry = GraphWALEntry::put_entity(writer.next_sequence(), 0, entity);
        
        let seq = writer.append(&entry).unwrap();
        assert_eq!(seq, entry.sequence);
        
        writer.sync().unwrap();
        writer.close().unwrap();
    }
    
    #[test]
    fn test_wal_reader_basic() {
        let dir = tempdir().unwrap();
        let metrics = Arc::new(WriteMetrics::new());
        
        // Write some entries
        {
            let config = GraphWALConfig::new(dir.path());
            let mut writer = GraphWALWriter::new(config, 0, metrics).unwrap();
            
            for i in 0..10 {
                let entity = Entity::new(&format!("Person {}", i), entity_types::PERSON);
                let entry = GraphWALEntry::put_entity(writer.next_sequence(), 0, entity);
                writer.append(&entry).unwrap();
            }
            
            writer.close().unwrap();
        }
        
        // Read entries
        let reader = GraphWALReader::new(dir.path());
        let entries = reader.read_all().unwrap();
        
        assert_eq!(entries.len(), 10);
        for entry in &entries {
            assert!(entry.verify_checksum());
        }
    }
    
    #[test]
    fn test_wal_replay() {
        let dir = tempdir().unwrap();
        let metrics = Arc::new(WriteMetrics::new());
        
        // Write entries
        {
            let config = GraphWALConfig::new(dir.path());
            let mut writer = GraphWALWriter::new(config, 0, metrics).unwrap();
            
            for i in 0..5 {
                let entity = Entity::new(&format!("Person {}", i), entity_types::PERSON);
                let entry = GraphWALEntry::put_entity(writer.next_sequence(), 0, entity);
                writer.append(&entry).unwrap();
            }
            
            writer.close().unwrap();
        }
        
        // Replay
        let reader = GraphWALReader::new(dir.path());
        let mut replayed_ids = Vec::new();
        
        let result = reader.replay(|entry| {
            if let Some(ref entity) = entry.entity {
                replayed_ids.push(entity.id.clone());
            }
            Ok(())
        }).unwrap();
        
        assert_eq!(result.entries_replayed, 5);
        assert_eq!(result.entries_skipped, 0);
        assert_eq!(replayed_ids.len(), 5);
    }
    
    #[test]
    fn test_wal_disabled() {
        let config = GraphWALConfig::disabled();
        let metrics = Arc::new(WriteMetrics::new());
        
        let mut writer = GraphWALWriter::new(config, 0, metrics).unwrap();
        assert!(!writer.is_enabled());
        
        // Operations should succeed but do nothing
        let entity = Entity::new("Test", entity_types::PERSON);
        let entry = GraphWALEntry::put_entity(1, 0, entity);
        let seq = writer.append(&entry).unwrap();
        assert_eq!(seq, 1);
    }
}
