//! Iroh stack: gossip + blobs + DHT auto-discovery.
//!
//! Architecture:
//!   - iroh-blobs FsStore: persistent content-addressed storage (TB-scale)
//!   - iroh-gossip: per-namespace Have/Want/Found broadcast
//!   - distributed-topic-tracker: mainline DHT bootstrap (zero config)
//!   - Endpoint: mDNS (LAN) + pkarr DHT + n0 relay (NAT traversal)
//!
//! No tickets. No join calls. No manual peer exchange.

use std::collections::HashMap;
use std::path::PathBuf;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::Arc;
use std::time::Duration;

use anyhow::{Context, Result};
use distributed_topic_tracker::{AutoDiscoveryGossip, RecordPublisher, TopicId as DttTopicId};
use iroh::address_lookup::mdns::MdnsAddressLookup;
use iroh::address_lookup::pkarr::dht::DhtAddressLookup;
use iroh::endpoint::RelayMode;
use iroh::protocol::Router;
use iroh::{Endpoint, EndpointId, SecretKey};
use iroh_blobs::store::fs::FsStore;
use iroh_blobs::{BlobFormat, BlobsProtocol, Hash, ALPN as BLOBS_ALPN};
use iroh_blobs::hashseq::HashSeq;
use iroh_gossip::api::Event;
use iroh_gossip::{Gossip, ALPN as GOSSIP_ALPN};
use serde::{Deserialize, Serialize};
use tokio::sync::{oneshot, RwLock};
use tracing::{debug, info, warn};

use crate::config;
use crate::eviction::EvictionIndex;

// ---------------------------------------------------------------------------
// Chunking config: data above threshold is stored as HashSeq (collection
// of content-addressed chunks). Identical chunks across entries are deduped
// automatically by the blob store. Configurable via env vars.
// ---------------------------------------------------------------------------

fn chunk_threshold() -> usize {
    std::env::var("IROHDS_CHUNK_THRESHOLD")
        .ok()
        .and_then(|v| v.parse().ok())
        .unwrap_or(4 * 1024 * 1024) // 4 MB
}

fn chunk_size() -> usize {
    std::env::var("IROHDS_CHUNK_SIZE")
        .ok()
        .and_then(|v| v.parse().ok())
        .unwrap_or(1024 * 1024) // 1 MB
}

/// Minimum compute cost (normalized ops) to announce on the gossip network.
/// Results below this are stored locally only. Default: 600 billion ops,
/// equivalent to ~15s on a 10-year-old 8-core desktop (~40 GFLOPS).
/// This is a universal constant: same value means the same amount of work
/// on every machine, regardless of hardware.
/// Override with IROHDS_MIN_OPS.
fn min_ops() -> u64 {
    std::env::var("IROHDS_MIN_OPS")
        .ok()
        .and_then(|v| v.parse().ok())
        .unwrap_or(600_000_000_000) // 600G ops = ~15s on a 40 GFLOPS machine
}

// ---------------------------------------------------------------------------
// CostMeta: machine-independent compute cost descriptor (mirrors coren FnCost)
// ---------------------------------------------------------------------------

#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize)]
pub struct CostMeta {
    #[serde(default)] pub ops: u64,
    #[serde(default)] pub mem_bytes: u64,
    #[serde(default)] pub peak_mem: u64,
    #[serde(default)] pub result_bytes: u64,
}

// ---------------------------------------------------------------------------
// Gossip protocol: 3 message types, JSON-encoded
// ---------------------------------------------------------------------------

#[derive(Debug, Clone, Serialize, Deserialize)]
enum ContentMsg {
    Have {
        key: [u8; 32], blob: [u8; 32],
        #[serde(default)] is_hashseq: bool,
        #[serde(default)] cost: CostMeta,
    },
    Want { key: [u8; 32], rid: u64 },
    Found {
        key: [u8; 32], blob: [u8; 32], rid: u64,
        #[serde(default)] is_hashseq: bool,
        #[serde(default)] cost: CostMeta,
    },
}

// ---------------------------------------------------------------------------
// Shared state: content index + pending request tracking
// ---------------------------------------------------------------------------

#[derive(Debug, Clone)]
struct ContentEntry {
    blob_hash: Hash,
    is_hashseq: bool,
    provider: Option<EndpointId>, // None = local
    cost: CostMeta,
}

pub struct SharedState {
    index: RwLock<HashMap<[u8; 32], ContentEntry>>,
    pending: RwLock<HashMap<u64, oneshot::Sender<(Hash, bool, EndpointId, CostMeta)>>>,
    /// Known gossip neighbors per namespace. Used by get_file to find
    /// peers that may have file data (HashSeqs) in their blob store.
    neighbors: RwLock<HashMap<String, std::collections::HashSet<EndpointId>>>,
    next_rid: AtomicU64,
}

impl SharedState {
    fn new() -> Self {
        Self {
            index: RwLock::new(HashMap::new()),
            pending: RwLock::new(HashMap::new()),
            neighbors: RwLock::new(HashMap::new()),
            next_rid: AtomicU64::new(1),
        }
    }

    async fn insert_local(&self, key: [u8; 32], hash: Hash, is_hashseq: bool, cost: CostMeta) {
        self.index.write().await.insert(
            key,
            ContentEntry { blob_hash: hash, is_hashseq, provider: None, cost },
        );
    }

    async fn insert_remote(&self, key: [u8; 32], hash: Hash, is_hashseq: bool,
                            from: EndpointId, cost: CostMeta) {
        let mut idx = self.index.write().await;
        if let Some(e) = idx.get(&key) {
            if e.provider.is_none() { return; } // don't overwrite local
        }
        idx.insert(key, ContentEntry { blob_hash: hash, is_hashseq, provider: Some(from), cost });
    }

    async fn lookup(&self, key: &[u8; 32]) -> Option<ContentEntry> {
        self.index.read().await.get(key).cloned()
    }

    fn next_rid(&self) -> u64 {
        self.next_rid.fetch_add(1, Ordering::Relaxed)
    }

    async fn register_wait(&self, rid: u64) -> oneshot::Receiver<(Hash, bool, EndpointId, CostMeta)> {
        let (tx, rx) = oneshot::channel();
        self.pending.write().await.insert(rid, tx);
        rx
    }

    async fn resolve(&self, rid: u64, hash: Hash, is_hashseq: bool,
                      from: EndpointId, cost: CostMeta) {
        if let Some(tx) = self.pending.write().await.remove(&rid) {
            let _ = tx.send((hash, is_hashseq, from, cost));
        }
    }
}

// ---------------------------------------------------------------------------
// Per-namespace handle: DTT topic + our gossip sender
// ---------------------------------------------------------------------------

struct NsHandle {
    _topic: distributed_topic_tracker::Topic, // kept alive: actor manages DHT publishing
    sender: distributed_topic_tracker::GossipSender,
}

// ---------------------------------------------------------------------------
// MemoNode
// ---------------------------------------------------------------------------

pub struct MemoNode {
    pub endpoint: Endpoint,
    secret_key: SecretKey,
    gossip: Gossip,
    blobs: FsStore,
    _router: Router,
    state: Arc<SharedState>,
    namespaces: RwLock<HashMap<String, NsHandle>>,
    evictions: RwLock<HashMap<String, EvictionIndex>>,
}

impl MemoNode {
    /// Start node with persistent FsStore and full discovery stack.
    pub async fn start(data_dir: PathBuf, no_relay: bool) -> Result<Self> {
        // Persistent secret key (stable node identity across restarts)
        let key_path = data_dir.join("secret.key");
        let secret_key = if key_path.exists() {
            let bytes = std::fs::read(&key_path).context("read secret key")?;
            let arr: [u8; 32] = bytes.try_into().map_err(|_| anyhow::anyhow!("bad key"))?;
            SecretKey::from_bytes(&arr)
        } else {
            let sk = SecretKey::generate(&mut rand::rng());
            std::fs::write(&key_path, sk.to_bytes())?;
            #[cfg(unix)]
            {
                use std::os::unix::fs::PermissionsExt;
                std::fs::set_permissions(&key_path, std::fs::Permissions::from_mode(0o600))?;
            }
            sk
        };

        // iroh 0.97: empty_builder() = blank canvas, configure everything explicitly
        let relay_mode = if no_relay { RelayMode::Disabled } else { RelayMode::Default };

        let endpoint = Endpoint::empty_builder()
            .secret_key(secret_key.clone())
            .relay_mode(relay_mode)
            .address_lookup(MdnsAddressLookup::builder())
            .address_lookup(DhtAddressLookup::builder())
            .bind()
            .await
            .context("bind endpoint")?;

        // Persistent blob store (supports TB-scale datasets)
        let blob_dir = data_dir.join("blobs");
        std::fs::create_dir_all(&blob_dir)?;
        #[cfg(unix)]
        {
            use std::os::unix::fs::PermissionsExt;
            std::fs::set_permissions(&blob_dir, std::fs::Permissions::from_mode(0o700))?;
        }
        let blobs = FsStore::load(&blob_dir).await.context("load blob store")?;

        let gossip = Gossip::builder().spawn(endpoint.clone());

        let router = Router::builder(endpoint.clone())
            .accept(BLOBS_ALPN, BlobsProtocol::new(&*blobs, None))
            .accept(GOSSIP_ALPN, gossip.clone())
            .spawn();

        info!(node_id = %endpoint.id().fmt_short(), "irohds node started (persistent)");

        Ok(Self {
            endpoint,
            secret_key,
            gossip,
            blobs,
            _router: router,
            state: Arc::new(SharedState::new()),
            namespaces: RwLock::new(HashMap::new()),
            evictions: RwLock::new(HashMap::new()),
        })
    }

    /// Load eviction index for a namespace and rebuild the in-memory content
    /// index from persisted blob_hash entries. No network I/O.
    /// Idempotent: returns immediately if already loaded.
    async fn ensure_eviction(&self, ns: &str) {
        if self.evictions.read().await.contains_key(ns) {
            return;
        }

        let ns_dir = config::namespace_dir(ns);
        let Ok(eviction) = EvictionIndex::load(&ns_dir) else { return };

        // Rebuild content index from persisted entries
        let mut rebuilt = 0usize;
        for (key_str, meta) in eviction.entries() {
            if meta.blob_hash.is_empty() { continue; }
            if let Ok(hash_bytes) = hex::decode(&meta.blob_hash) {
                if let Ok(arr) = <[u8; 32]>::try_from(hash_bytes.as_slice()) {
                    self.state.insert_local(
                        key_to_32(key_str), Hash::from_bytes(arr), meta.is_hashseq,
                        CostMeta {
                            ops: meta.ops, mem_bytes: meta.mem_bytes,
                            peak_mem: meta.peak_mem, result_bytes: meta.size_bytes,
                        },
                    ).await;
                    rebuilt += 1;
                }
            }
        }
        if rebuilt > 0 {
            info!(ns, rebuilt, "content index rebuilt from eviction.json");
        }

        self.evictions.write().await.insert(ns.to_string(), eviction);
    }

    /// Join a namespace via DHT auto-discovery (zero config).
    /// Calls ensure_eviction first, then does the network join.
    async fn ensure_ns(self: &Arc<Self>, ns: &str) -> Result<()> {
        if self.namespaces.read().await.contains_key(ns) {
            return Ok(());
        }

        // Load eviction + rebuild content index (no network)
        self.ensure_eviction(ns).await;

        let ns_dir = config::namespace_dir(ns);

        // DTT: derive ed25519 keys from iroh secret key
        let signing_key = ed25519_dalek::SigningKey::from_bytes(&self.secret_key.to_bytes());
        let verifying_key = signing_key.verifying_key();

        // Deterministic secret: all peers in same namespace derive same DHT keys
        let initial_secret = blake3::hash(
            format!("irohds-ns-secret:{ns}").as_bytes()
        ).as_bytes().to_vec();

        let topic_id = DttTopicId::new(format!("irohds:{ns}"));
        let record_publisher = RecordPublisher::new(
            topic_id,
            verifying_key,
            signing_key,
            None,
            initial_secret,
        );

        // Subscribe with DHT auto-discovery (non-blocking bootstrap)
        let dtt_topic = self.gossip
            .subscribe_and_join_with_auto_discovery_no_wait(record_publisher)
            .await
            .context("DTT subscribe")?;

        let (sender, receiver) = dtt_topic.split().await?;

        // Fallback: join configured bootstrap peers (for DHT-blocked networks)
        let global_cfg = config::load_global_config().unwrap_or_default();
        if !global_cfg.bootstrap_peers.is_empty() {
            let mut peers = Vec::new();
            for hex_id in &global_cfg.bootstrap_peers {
                if let Ok(bytes) = hex::decode(hex_id) {
                    if let Ok(arr) = <[u8; 32]>::try_from(bytes.as_slice()) {
                        if let Ok(pk) = iroh::PublicKey::from_bytes(&arr) {
                            peers.push(pk);
                        }
                    }
                }
            }
            if !peers.is_empty() {
                info!(ns, count = peers.len(), "joining bootstrap peers");
                let _ = sender.join_peers(peers, None).await;
            }
        }

        self.namespaces.write().await.insert(ns.to_string(), NsHandle {
            _topic: dtt_topic.clone(),
            sender: sender.clone(),
        });

        // Spawn gossip event processor
        let state = self.state.clone();
        let ns_owned = ns.to_string();
        let sender_for_task = sender;
        tokio::spawn(async move {
            loop {
                let Some(item) = receiver.next().await else { break };
                let Ok(event) = item else { continue };
                match event {
                    Event::Received(msg) => {
                        let Ok(content) = serde_json::from_slice::<ContentMsg>(&msg.content) else {
                            continue;
                        };
                        let from = msg.delivered_from;
                        match content {
                            ContentMsg::Have { key, blob, is_hashseq, cost } => {
                                state.insert_remote(key, Hash::from_bytes(blob), is_hashseq, from, cost).await;
                                debug!(ns = %ns_owned, is_hashseq, "peer has content");
                            }
                            ContentMsg::Want { key, rid } => {
                                if let Some(e) = state.lookup(&key).await {
                                    if e.provider.is_none() {
                                        let resp = ContentMsg::Found {
                                            key,
                                            blob: *e.blob_hash.as_bytes(),
                                            rid,
                                            is_hashseq: e.is_hashseq,
                                            cost: e.cost,
                                        };
                                        if let Ok(encoded) = serde_json::to_vec(&resp) {
                                            let _ = sender_for_task.broadcast(encoded).await;
                                        }
                                    }
                                }
                            }
                            ContentMsg::Found { key, blob, rid, is_hashseq, cost } => {
                                let hash = Hash::from_bytes(blob);
                                state.insert_remote(key, hash, is_hashseq, from, cost).await;
                                state.resolve(rid, hash, is_hashseq, from, cost).await;
                            }
                        }
                    }
                    Event::NeighborUp(id) => {
                        state.neighbors.write().await
                            .entry(ns_owned.clone())
                            .or_default()
                            .insert(id);
                        debug!(ns = %ns_owned, peer = %id.fmt_short(), "peer joined");
                    }
                    Event::NeighborDown(id) => {
                        if let Some(set) = state.neighbors.write().await.get_mut(&*ns_owned) {
                            set.remove(&id);
                        }
                        debug!(ns = %ns_owned, peer = %id.fmt_short(), "peer left");
                    }
                    Event::Lagged => {
                        warn!(ns = %ns_owned, "gossip lagged");
                    }
                }
            }
        });

        info!(ns, "namespace active (DHT discovery)");
        Ok(())
    }

    /// Store a cached result and announce to peers.
    ///
    /// Large values (> IROHDS_CHUNK_THRESHOLD, default 4MB) are split into
    /// fixed-size chunks stored as individual blobs, referenced by an iroh
    /// HashSeq (native collection type). Identical chunks across entries
    /// are deduped automatically by the BLAKE3 content-addressed store.
    ///
    /// For the 1TB-CSV-with-one-row-appended scenario: peer downloads
    /// ~32KB manifest + ~1MB new final chunk instead of re-downloading 1TB.
    pub async fn put(
        self: &Arc<Self>, ns: &str, key_hex: &str,
        data: &[u8], compute_ns: u64, cost: CostMeta, tag: &str,
    ) -> Result<()> {
        self.ensure_ns(ns).await?;
        let key = key_to_32(key_hex);

        let threshold = chunk_threshold();
        let (hash, is_hashseq) = if data.len() > threshold {
            // Split into fixed chunks, store each as an independent raw blob
            let cs = chunk_size();
            let mut chunk_hashes = Vec::with_capacity(data.len() / cs + 1);
            for piece in data.chunks(cs) {
                let info = self.blobs.blobs().add_slice(piece).await?;
                chunk_hashes.push(info.hash);
            }

            // Build a HashSeq (iroh's native collection: just concatenated hashes)
            let seq: HashSeq = chunk_hashes.iter().collect();
            let seq_bytes: bytes::Bytes = seq.into();
            let info = self.blobs.blobs()
                .add_bytes_with_opts((seq_bytes, BlobFormat::HashSeq))
                .await?;

            debug!(ns, key = key_hex, chunks = chunk_hashes.len(),
                   "stored as HashSeq");
            (info.hash, true)
        } else {
            let info = self.blobs.blobs().add_slice(data).await?;
            (info.hash, false)
        };

        let cost = CostMeta { result_bytes: data.len() as u64, ..cost };
        self.state.insert_local(key, hash, is_hashseq, cost).await;

        if let Some(ev) = self.evictions.write().await.get_mut(ns) {
            let meta = crate::eviction::EntryMeta::new(
                compute_ns, cost.ops, cost.mem_bytes, cost.peak_mem, data.len() as u64, tag,
            ).with_blob(&hex::encode(hash.as_bytes()), is_hashseq);
            ev.record(key_hex, meta);
        }

        // Only announce to network if normalized compute cost exceeds threshold
        if cost.ops >= min_ops() {
            let msg = ContentMsg::Have {
                key, blob: *hash.as_bytes(), is_hashseq, cost,
            };
            if let Ok(encoded) = serde_json::to_vec(&msg) {
                if let Some(handle) = self.namespaces.read().await.get(ns) {
                    let _ = handle.sender.broadcast(encoded).await;
                }
            }
            debug!(ns, key = key_hex, "stored + announced");
        } else {
            debug!(ns, key = key_hex, ops = cost.ops, "stored locally (below threshold)");
        }
        Ok(())
    }

    /// Look up: local -> known remote -> gossip broadcast Want.
    /// HashSeq entries are resolved transparently: only missing chunks are fetched.
    pub async fn get(self: &Arc<Self>, ns: &str, key_hex: &str) -> Result<Option<Vec<u8>>> {
        let key = key_to_32(key_hex);

        // Load eviction index + rebuild content index (no network)
        self.ensure_eviction(ns).await;

        if let Some(ev) = self.evictions.write().await.get_mut(ns) {
            ev.touch(key_hex);
        }

        // 1. Local blob -- no network needed
        if let Some(entry) = self.state.lookup(&key).await {
            if entry.provider.is_none() {
                return self.read_entry(&entry, None).await;
            }
        }

        // 2. Need network from here -- join namespace
        self.ensure_ns(ns).await?;

        // 2. Known remote provider
        if let Some(entry) = self.state.lookup(&key).await {
            if let Some(peer) = entry.provider {
                match self.fetch_and_store(&entry, peer).await {
                    Ok(data) => {
                        self.state.insert_local(key, entry.blob_hash, entry.is_hashseq, entry.cost).await;
                        return Ok(Some(data));
                    }
                    Err(e) => debug!("fetch from known provider failed: {e}"),
                }
            }
        }

        // 3. Ask the swarm
        let rid = self.state.next_rid();
        let rx = self.state.register_wait(rid).await;

        let msg = ContentMsg::Want { key, rid };
        if let Ok(encoded) = serde_json::to_vec(&msg) {
            if let Some(handle) = self.namespaces.read().await.get(ns) {
                let _ = handle.sender.broadcast(encoded).await;
            }
        }

        let timeout_secs: u64 = std::env::var("IROHDS_GET_TIMEOUT")
            .ok()
            .and_then(|v| v.parse().ok())
            .unwrap_or(15);

        match tokio::time::timeout(Duration::from_secs(timeout_secs), rx).await {
            Ok(Ok((hash, is_hashseq, peer, cost))) => {
                let entry = ContentEntry {
                    blob_hash: hash, is_hashseq, provider: Some(peer), cost,
                };
                match self.fetch_and_store(&entry, peer).await {
                    Ok(data) => {
                        self.state.insert_local(key, hash, is_hashseq, cost).await;
                        Ok(Some(data))
                    }
                    Err(_) => Ok(None),
                }
            }
            _ => Ok(None),
        }
    }

    /// Metadata-only probe: returns CostMeta if known.
    /// Does NOT transfer blob data or broadcast Want to the swarm.
    pub async fn probe(&self, ns: &str, key_hex: &str) -> Result<Option<CostMeta>> {
        // Load eviction + rebuild content index (no network, idempotent)
        self.ensure_eviction(ns).await;
        let key = key_to_32(key_hex);

        if let Some(entry) = self.state.lookup(&key).await {
            let size = if entry.provider.is_none() {
                self.blobs.blobs().get_bytes(entry.blob_hash).await
                    .map(|b| b.len() as u64).unwrap_or(0)
            } else {
                entry.cost.result_bytes
            };

            // Use ContentEntry cost (populated by gossip for remote,
            // by insert_local for local). Fall back to eviction for
            // old entries that predate the CostMeta addition.
            let cost = if entry.cost.ops > 0 {
                CostMeta { result_bytes: size, ..entry.cost }
            } else {
                self.evictions.read().await
                    .get(ns)
                    .and_then(|ev| ev.get_meta(key_hex))
                    .map(|m| CostMeta { ops: m.ops, mem_bytes: m.mem_bytes, peak_mem: m.peak_mem, result_bytes: size })
                    .unwrap_or(CostMeta { result_bytes: size, ..Default::default() })
            };

            return Ok(Some(cost));
        }
        Ok(None)
    }

    /// Combined probe+get for local data. One IPC round-trip.
    /// Returns Some(bytes) if cached locally, None if miss.
    /// No network I/O, no gossip, no DHT. Just eviction.json + blob store.
    pub async fn check_local(&self, ns: &str, key_hex: &str) -> Result<Option<Vec<u8>>> {
        self.ensure_eviction(ns).await;
        let key = key_to_32(key_hex);

        if let Some(ev) = self.evictions.write().await.get_mut(ns) {
            ev.touch(key_hex);
        }

        if let Some(entry) = self.state.lookup(&key).await {
            if entry.provider.is_none() {
                return self.read_entry(&entry, None).await;
            }
        }
        Ok(None)
    }

    /// Read an entry's data. For HashSeq, reassemble from local chunks.
    async fn read_entry(
        &self, entry: &ContentEntry, provider: Option<EndpointId>,
    ) -> Result<Option<Vec<u8>>> {
        if !entry.is_hashseq {
            return Ok(self.blobs.blobs().get_bytes(entry.blob_hash).await
                .ok().map(|b| b.to_vec()));
        }

        // Read the HashSeq manifest, then concatenate children
        let seq_bytes = self.blobs.blobs().get_bytes(entry.blob_hash).await?;
        let seq = HashSeq::new(seq_bytes).context("invalid HashSeq")?;

        let mut result = Vec::new();
        for chunk_hash in seq.iter() {
            match self.blobs.blobs().get_bytes(chunk_hash).await {
                Ok(bytes) => result.extend_from_slice(&bytes),
                Err(_) if provider.is_some() => {
                    // Chunk missing locally but we have a provider
                    let data = self.fetch_raw_blob(chunk_hash, provider.unwrap()).await?;
                    result.extend_from_slice(&data);
                }
                Err(e) => return Err(e.into()),
            }
        }
        Ok(Some(result))
    }

    /// Fetch a remote entry and store locally. For HashSeq, only fetches missing chunks.
    async fn fetch_and_store(
        &self, entry: &ContentEntry, peer: EndpointId,
    ) -> Result<Vec<u8>> {
        if !entry.is_hashseq {
            return self.fetch_raw_blob(entry.blob_hash, peer).await;
        }

        // Fetch the HashSeq manifest (small: N * 32 bytes)
        let seq_bytes = match self.blobs.blobs().get_bytes(entry.blob_hash).await {
            Ok(b) => b,
            Err(_) => {
                let data = self.fetch_raw_blob(entry.blob_hash, peer).await?;
                let data_bytes: bytes::Bytes = data.clone().into();
                let _ = self.blobs.blobs()
                    .add_bytes_with_opts((data_bytes, BlobFormat::HashSeq))
                    .await?;
                data.into()
            }
        };
        let seq = HashSeq::new(seq_bytes).context("invalid HashSeq")?;

        // Fetch only missing children
        let mut result = Vec::new();
        let mut local_count = 0usize;
        let mut fetched_count = 0usize;

        for chunk_hash in seq.iter() {
            match self.blobs.blobs().get_bytes(chunk_hash).await {
                Ok(bytes) => {
                    result.extend_from_slice(&bytes);
                    local_count += 1;
                }
                Err(_) => {
                    let bytes = self.fetch_raw_blob(chunk_hash, peer).await
                        .context("fetch chunk")?;
                    let _ = self.blobs.blobs().add_slice(&bytes).await;
                    result.extend_from_slice(&bytes);
                    fetched_count += 1;
                }
            }
        }

        debug!(local = local_count, fetched = fetched_count,
               total = local_count + fetched_count, "HashSeq resolved");
        Ok(result)
    }

    /// Download a single raw blob from a peer.
    async fn fetch_raw_blob(&self, hash: Hash, peer: EndpointId) -> Result<Vec<u8>> {
        let dl = self.blobs.downloader(&self.endpoint);
        dl.download(hash, vec![peer]).await?;
        Ok(self.blobs.blobs().get_bytes(hash).await?.to_vec())
    }

    // -- File-based operations (streaming, never loads full file into RAM) --

    /// Import a file from disk into the blob store as a HashSeq of chunks.
    /// Returns (hashseq_hash_hex, file_size).
    ///
    /// Does NOT gossip-announce -- the blobs protocol handler automatically
    /// serves stored blobs to any peer that requests them by hash. The
    /// caller (Python decorator) stores the hash in a FileRef pickle and
    /// puts that via the normal put() path, which handles gossip.
    pub async fn put_file(
        self: &Arc<Self>, ns: &str, key_hex: &str,
        path: &std::path::Path, compute_ns: u64, cost: CostMeta, tag: &str,
    ) -> Result<(String, u64)> {
        self.ensure_ns(ns).await?;

        let file_size = tokio::fs::metadata(path).await
            .with_context(|| format!("stat {}", path.display()))?.len();

        let cs = chunk_size();
        let mut chunk_hashes = Vec::with_capacity((file_size as usize / cs) + 1);

        // Stream file in fixed chunks -- bounded memory usage
        use tokio::io::AsyncReadExt;
        let mut file = tokio::fs::File::open(path).await
            .with_context(|| format!("open {}", path.display()))?;
        let mut buf = vec![0u8; cs];
        loop {
            let mut filled = 0;
            while filled < cs {
                let n = file.read(&mut buf[filled..]).await?;
                if n == 0 { break; }
                filled += n;
            }
            if filled == 0 { break; }
            let info = self.blobs.blobs().add_slice(&buf[..filled]).await?;
            chunk_hashes.push(info.hash);
        }

        let seq: HashSeq = chunk_hashes.iter().collect();
        let seq_bytes: bytes::Bytes = seq.into();
        let info = self.blobs.blobs()
            .add_bytes_with_opts((seq_bytes, BlobFormat::HashSeq))
            .await?;
        let hash = info.hash;
        let hash_hex = hex::encode(hash.as_bytes());

        if let Some(ev) = self.evictions.write().await.get_mut(ns) {
            let meta = crate::eviction::EntryMeta::new(
                compute_ns, cost.ops, cost.mem_bytes, cost.peak_mem, file_size, tag,
            ).with_blob(&hash_hex, true);
            ev.record(key_hex, meta);
        }

        debug!(ns, key = key_hex, chunks = chunk_hashes.len(),
               size = file_size, hash = %hash_hex, "file stored as HashSeq");
        Ok((hash_hex, file_size))
    }

    /// Materialize a HashSeq (identified by blob hash) to a file on disk.
    /// Downloads missing chunks from namespace neighbors as needed.
    /// Streams directly to disk -- never holds more than one chunk in RAM.
    pub async fn get_file(
        self: &Arc<Self>, ns: &str, blob_hash_hex: &str,
        path: &std::path::Path,
    ) -> Result<Option<u64>> {
        self.ensure_ns(ns).await?;

        let hash_bytes = hex::decode(blob_hash_hex).context("invalid hex")?;
        let hash_arr: [u8; 32] = hash_bytes.try_into()
            .map_err(|_| anyhow::anyhow!("hash must be 32 bytes"))?;
        let hash = Hash::from_bytes(hash_arr);

        // Collect namespace neighbors as potential providers
        let neighbors: Vec<EndpointId> = self.state.neighbors.read().await
            .get(ns)
            .map(|s| s.iter().copied().collect())
            .unwrap_or_default();

        // Try to get the HashSeq manifest (locally or from neighbors)
        let seq_bytes = match self.blobs.blobs().get_bytes(hash).await {
            Ok(b) => b,
            Err(_) => {
                // Download from first available neighbor
                let mut downloaded = false;
                for peer in &neighbors {
                    if let Ok(()) = self.blobs.downloader(&self.endpoint)
                        .download(hash, vec![*peer]).await
                    {
                        downloaded = true;
                        break;
                    }
                }
                if !downloaded {
                    debug!(ns, hash = blob_hash_hex, "HashSeq not found locally or from peers");
                    return Ok(None);
                }
                self.blobs.blobs().get_bytes(hash).await?
            }
        };

        let seq = match HashSeq::new(seq_bytes) {
            Some(s) => s,
            None => anyhow::bail!("blob is not a valid HashSeq"),
        };

        // Stream chunks to file
        use tokio::io::AsyncWriteExt;
        if let Some(parent) = path.parent() {
            tokio::fs::create_dir_all(parent).await.ok();
        }
        let mut file = tokio::fs::File::create(path).await
            .with_context(|| format!("create {}", path.display()))?;
        let mut total = 0u64;
        let mut local_count = 0usize;
        let mut fetched_count = 0usize;

        for chunk_hash in seq.iter() {
            let chunk_data = match self.blobs.blobs().get_bytes(chunk_hash).await {
                Ok(b) => { local_count += 1; b.to_vec() }
                Err(_) => {
                    // Try neighbors
                    let mut got = None;
                    for peer in &neighbors {
                        if self.blobs.downloader(&self.endpoint)
                            .download(chunk_hash, vec![*peer]).await.is_ok()
                        {
                            if let Ok(b) = self.blobs.blobs().get_bytes(chunk_hash).await {
                                got = Some(b.to_vec());
                                break;
                            }
                        }
                    }
                    match got {
                        Some(d) => { fetched_count += 1; d }
                        None => anyhow::bail!("chunk {} unavailable", hex::encode(chunk_hash.as_bytes())),
                    }
                }
            };
            file.write_all(&chunk_data).await?;
            total += chunk_data.len() as u64;
        }
        file.flush().await?;

        debug!(ns, local = local_count, fetched = fetched_count,
               total_bytes = total, "file materialized from HashSeq");
        Ok(Some(total))
    }

    // -- Public info/management --

    /// Pre-join a namespace (starts DHT discovery without get/put).
    pub async fn join(self: &Arc<Self>, ns: &str) -> Result<()> {
        self.ensure_ns(ns).await
    }

    pub fn node_id(&self) -> String {
        self.endpoint.id().fmt_short().to_string()
    }

    pub fn node_id_hex(&self) -> String {
        hex::encode(self.endpoint.id().as_bytes())
    }

    pub async fn ns_count(&self) -> usize {
        self.namespaces.read().await.len()
    }

    pub async fn namespace_names(&self) -> Vec<String> {
        self.namespaces.read().await.keys().cloned().collect()
    }

    pub async fn evict_tag(&self, ns: &str, tag_prefix: &str) -> Result<Vec<String>> {
        if let Some(ev) = self.evictions.write().await.get_mut(ns) {
            let keys = ev.evict_by_tag(tag_prefix);
            ev.persist()?;
            Ok(keys)
        } else {
            Ok(vec![])
        }
    }

    pub async fn run_eviction(&self, ns: &str) -> Result<()> {
        let cfg = config::load_ns_config(ns)?;
        if let Some(ev) = self.evictions.write().await.get_mut(ns) {
            let victims = ev.eviction_candidates(cfg.max_cache_bytes);
            for k in &victims { ev.remove(k); }
            if !victims.is_empty() {
                ev.persist()?;
                info!(ns, evicted = victims.len(), "eviction pass");
            }
        }
        Ok(())
    }

    /// Persist eviction indices on shutdown.
    pub async fn shutdown(&self) {
        for (_, ev) in self.evictions.write().await.iter() {
            let _ = ev.persist();
        }
        info!("eviction state persisted");
    }
}

/// Derive a 32-byte gossip identifier from an arbitrary key string.
///
/// Keys may be pure hex ("abcdef01...") or tagged ("module.func/abcdef01...").
/// blake3 hashing makes the gossip protocol key-format-agnostic: any string
/// deterministically maps to [u8; 32], so peers with the same key agree.
fn key_to_32(s: &str) -> [u8; 32] {
    *blake3::hash(s.as_bytes()).as_bytes()
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn key_to_32_deterministic() {
        let a = key_to_32("mymodule.func/abc123");
        let b = key_to_32("mymodule.func/abc123");
        assert_eq!(a, b);
    }

    #[test]
    fn key_to_32_different_keys_differ() {
        let a = key_to_32("mod.f/aaa");
        let b = key_to_32("mod.f/bbb");
        assert_ne!(a, b);
    }

    #[test]
    fn content_msg_serde() {
        let msg = ContentMsg::Have {
            key: [1u8; 32], blob: [2u8; 32], is_hashseq: false,
            cost: CostMeta { ops: 100, mem_bytes: 200, peak_mem: 300, result_bytes: 400 },
        };
        let bytes = serde_json::to_vec(&msg).unwrap();
        let decoded: ContentMsg = serde_json::from_slice(&bytes).unwrap();
        match decoded {
            ContentMsg::Have { key, blob, cost, .. } => {
                assert_eq!(key, [1u8; 32]);
                assert_eq!(blob, [2u8; 32]);
                assert_eq!(cost.ops, 100);
                assert_eq!(cost.result_bytes, 400);
            }
            _ => panic!("wrong variant"),
        }
    }

    #[test]
    fn content_msg_hashseq_flag() {
        let msg = ContentMsg::Have {
            key: [1u8; 32], blob: [2u8; 32], is_hashseq: true,
            cost: CostMeta::default(),
        };
        let bytes = serde_json::to_vec(&msg).unwrap();
        let decoded: ContentMsg = serde_json::from_slice(&bytes).unwrap();
        match decoded {
            ContentMsg::Have { is_hashseq, .. } => assert!(is_hashseq),
            _ => panic!("wrong variant"),
        }
    }

    fn test_peer() -> EndpointId {
        iroh::SecretKey::from_bytes(&[42u8; 32]).public()
    }

    #[tokio::test]
    async fn shared_state_local_not_overwritten() {
        let state = SharedState::new();
        let hash = Hash::from_bytes([1u8; 32]);
        let cost = CostMeta { ops: 100, mem_bytes: 200, peak_mem: 300, result_bytes: 400 };
        state.insert_local([0u8; 32], hash, false, cost).await;

        let remote = test_peer();
        let cost2 = CostMeta { ops: 999, mem_bytes: 999, peak_mem: 999, result_bytes: 999 };
        state.insert_remote([0u8; 32], Hash::from_bytes([2u8; 32]), false, remote, cost2).await;

        let e = state.lookup(&[0u8; 32]).await.unwrap();
        assert!(e.provider.is_none());
        assert_eq!(e.blob_hash, hash);
        assert_eq!(e.cost.ops, 100); // local values preserved
    }

    #[tokio::test]
    async fn pending_resolve() {
        let state = SharedState::new();
        let rid = state.next_rid();
        let rx = state.register_wait(rid).await;

        let hash = Hash::from_bytes([1u8; 32]);
        let peer = test_peer();
        let cost = CostMeta { ops: 10, mem_bytes: 20, peak_mem: 30, result_bytes: 40 };
        state.resolve(rid, hash, false, peer, cost).await;

        let (h, is_hs, _p, c) = rx.await.unwrap();
        assert_eq!(h, hash);
        assert!(!is_hs);
        assert_eq!(c.ops, 10);
    }

    #[tokio::test]
    async fn pending_resolve_hashseq() {
        let state = SharedState::new();
        let rid = state.next_rid();
        let rx = state.register_wait(rid).await;

        let hash = Hash::from_bytes([1u8; 32]);
        let peer = test_peer();
        state.resolve(rid, hash, true, peer, CostMeta::default()).await;

        let (_, is_hs, _, _) = rx.await.unwrap();
        assert!(is_hs);
    }
}
