//! JSON parsing and gzip decompression for GDELT Web NGrams files.

use flate2::read::GzDecoder;
use serde::Deserialize;
use std::collections::HashMap;
use std::fs::File;
use std::io::{BufRead, BufReader, Read};
use std::path::Path;

/// A single raw GDELT Web NGrams entry as it appears in the JSON.
#[derive(Debug, Deserialize)]
#[allow(dead_code)]
pub struct RawEntry {
    #[serde(default)]
    pub date: String,
    #[serde(default)]
    pub ngram: String,
    #[serde(default)]
    pub lang: String,
    #[serde(default, rename = "type")]
    pub lang_type: u8,
    #[serde(default)]
    pub pos: i32,
    #[serde(default)]
    pub pre: String,
    #[serde(default)]
    pub post: String,
    #[serde(default)]
    pub url: String,
}

/// Parsed and grouped entries ready for fragment creation.
/// Maps URL -> Vec<RawEntry>, preserving first-seen URL order.
pub struct GroupedEntries {
    pub groups: HashMap<String, Vec<RawEntry>>,
    pub url_order: Vec<String>,
}

/// Open a file, auto-detecting gzip vs plain JSON based on extension.
fn open_reader(path: &Path) -> Box<dyn Read> {
    let file = File::open(path).expect("Failed to open input file");
    let ext = path.extension().and_then(|e| e.to_str()).unwrap_or("");
    if ext == "gz" {
        Box::new(GzDecoder::new(file))
    } else {
        Box::new(file)
    }
}

/// Load and filter a GDELT Web NGrams file (gzipped or plain JSON).
///
/// Filters by language and URL substrings. Groups entries by URL.
pub fn load_and_filter(
    path: &Path,
    language: Option<&str>,
    url_filters: Option<&[String]>,
) -> GroupedEntries {
    let reader = open_reader(path);
    let buf = BufReader::with_capacity(256 * 1024, reader);

    let mut groups: HashMap<String, Vec<RawEntry>> = HashMap::new();
    let mut url_order: Vec<String> = Vec::new();

    for line in buf.lines() {
        let line = match line {
            Ok(l) => l,
            Err(_) => continue,
        };
        if line.is_empty() {
            continue;
        }

        let entry: RawEntry = match serde_json::from_str(&line) {
            Ok(e) => e,
            Err(_) => continue,
        };

        // Language filter
        if let Some(lang) = language {
            if entry.lang != lang {
                continue;
            }
        }

        // Skip entries without URL
        if entry.url.is_empty() {
            continue;
        }

        // URL filter: at least one substring must match
        if let Some(filters) = url_filters {
            if !filters.iter().any(|f| entry.url.contains(f.as_str())) {
                continue;
            }
        }

        if !groups.contains_key(&entry.url) {
            url_order.push(entry.url.clone());
        }
        groups.entry(entry.url.clone()).or_default().push(entry);
    }

    GroupedEntries { groups, url_order }
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::io::Write;
    use tempfile::NamedTempFile;

    fn make_entry_json(url: &str, ngram: &str, pos: i32, pre: &str, post: &str) -> String {
        format!(
            r#"{{"date":"2020-01-01","ngram":"{}","lang":"en","type":1,"pos":{},"pre":"{}","post":"{}","url":"{}"}}"#,
            ngram, pos, pre, post, url
        )
    }

    #[test]
    fn test_load_and_filter_basic() {
        let mut f = NamedTempFile::with_suffix(".json").unwrap();
        writeln!(f, "{}", make_entry_json("http://example.com/a", "hello", 0, "pre", "post")).unwrap();
        writeln!(f, "{}", make_entry_json("http://example.com/a", "world", 10, "pre2", "post2")).unwrap();
        writeln!(f, "{}", make_entry_json("http://other.com/b", "foo", 0, "pre3", "post3")).unwrap();

        let result = load_and_filter(f.path(), Some("en"), None);
        assert_eq!(result.url_order.len(), 2);
        assert_eq!(result.groups["http://example.com/a"].len(), 2);
        assert_eq!(result.groups["http://other.com/b"].len(), 1);
    }

    #[test]
    fn test_url_filter() {
        let mut f = NamedTempFile::with_suffix(".json").unwrap();
        writeln!(f, "{}", make_entry_json("http://example.com/a", "hello", 0, "pre", "post")).unwrap();
        writeln!(f, "{}", make_entry_json("http://other.com/b", "foo", 0, "pre3", "post3")).unwrap();

        let filters = vec!["example.com".to_string()];
        let result = load_and_filter(f.path(), Some("en"), Some(&filters));
        assert_eq!(result.url_order.len(), 1);
        assert!(result.groups.contains_key("http://example.com/a"));
    }
}
