from collections import Counter
import datetime as dt
from pathlib import Path
import shutil
import unittest

import polars as pl

from shantay.dsa_sor import StatementsOfReasons
from shantay.metadata import Metadata
from shantay.model import Coverage, Daily, Storage
from shantay.processor import Processor
from shantay.stats import Collector, Statistics
from shantay.tool import configure_logging

ROOT = Path(__file__).parent
FIXTURE = ROOT / "fixture"
ARCHIVE = FIXTURE / "archive"

# We never copy the parquet files out of staging.
# So we only need ARCHIVE and STAGING.
STAGING = ROOT / "tmp"
LOGFILE = STAGING / "log.log"
SENTINEL = STAGING / "prepare.run"

ZIP_FILES = [
    "sor-global-2024-03-14-full-00000.csv.zip",
    "sor-global-2024-03-14-full-00001.csv.zip",
]

CSV_FILES = [
    "sor-global-2024-03-14-full-00000-00000.csv",
    "sor-global-2024-03-14-full-00000-00001.csv",
    "sor-global-2024-03-14-full-00001-00000.csv",
    "sor-global-2024-03-14-full-00001-00001.csv",
]

FILTER = "STATEMENT_CATEGORY_PROTECTION_OF_MINORS"


def setUpModule():
    # Since the staging directory and log file are shared across test modules,
    # we use per-test-module sentinel files to detect new runs.
    if SENTINEL.exists():
        shutil.rmtree(STAGING)
    STAGING.mkdir(exist_ok=True)
    SENTINEL.write_text(f"{dt.datetime.now()}\n")

    configure_logging(str(LOGFILE), verbose=True)


def tearDownModule():
    pass


class TestPrepare(unittest.TestCase):

    def assertFileEqual(self, path1: Path, path2: Path) -> None:
        data1 = path1.read_bytes()
        data2 = path2.read_bytes()
        self.assertEqual(len(data1), len(data2), "file contents must have equal length")
        self.assertEqual(data1, data2, "file contents must be equal")

    def test_extraction(self):
        self.maxDiff = None  # When something goes wrong, we want to see *all* about it
        with self.subTest("set up metadata, runner, and release"):
            dataset = StatementsOfReasons()
            storage = Storage(
                archive_root=ARCHIVE, working_root=STAGING, staging_root=STAGING
            )
            release = Daily(2024, 3, 14)
            coverage = Coverage(release, release, FILTER)
            metadata = Metadata(FILTER, {})
            processor = Processor(
                dataset=dataset,
                storage=storage,
                coverage=coverage,
                metadata=metadata,
            )

            digest = release.parent_directory / dataset.digest_name(release)
            archive = release.parent_directory / dataset.archive_name(release)

        with self.subTest("stage archive by copying fixture"):
            self.assertFalse((STAGING / digest).exists())
            self.assertFalse((STAGING / archive).exists())
            processor.stage_archive(release)
            self.assertTrue((STAGING / digest).exists())
            self.assertTrue((STAGING / archive).exists())
            self.assertFileEqual(ARCHIVE / digest, STAGING / digest)
            self.assertFileEqual(ARCHIVE / archive, STAGING / archive)

        with self.subTest("determine archived files"):
            filenames = processor.list_archived_files(STAGING, release)
            self.assertListEqual(filenames, ZIP_FILES)

        with self.subTest("unarchive first of two CSV files"):
            workdir = STAGING / release.temp_directory
            self.assertFalse(workdir.exists())

            processor.unarchive_file(STAGING, release, 0, ZIP_FILES[0])
            self.assertTrue(workdir.exists())
            self.assertListEqual(
                sorted(p.name for p in workdir.glob("*")), CSV_FILES[:2]
            )
            self.assertFileEqual(workdir / CSV_FILES[0], FIXTURE / "csv" / CSV_FILES[0])
            self.assertFileEqual(workdir / CSV_FILES[1], FIXTURE / "csv" / CSV_FILES[1])

        with self.subTest("determine row counts"):
            glob = f"{STAGING / release.temp_directory}/*.csv"
            count1, count2 = dataset._extract_row_counts(glob, 0, ZIP_FILES[0])
            self.assertEqual(count1, 100)
            self.assertEqual(count2, 12)

        with self.subTest("extract first batch of category data"):
            frame = dataset._extract_filtered_rows(
                csv_files=glob,
                release=release,
                index=0,
                name=ZIP_FILES[0],
                filter=FILTER,
            )
            dataset._validate_schema(frame)

            counters = Counter(batch_count=2)
            counters += dataset._assemble_frame_counters(frame, count1, count2)
            self.assertEqual(counters["batch_count"], 2)
            self.assertEqual(counters["total_rows"], 100)
            self.assertEqual(counters["total_rows_with_keywords"], 12)
            self.assertEqual(counters["batch_rows"], 8)
            self.assertEqual(counters["batch_rows_with_keywords"], 2)

            framedir = STAGING / release.directory
            self.assertFalse(framedir.exists())
            framedir.mkdir(parents=True)
            batch0 = framedir / release.batch_file(0)
            frame.write_parquet(batch0)

            self.assertFileEqual(batch0, FIXTURE / release.batch_file(0))

        with self.subTest("extract second batch of category data"):
            batch1 = STAGING / release.directory / release.batch_file(1)
            self.assertFalse(batch1.exists())

            processor.unarchive_file(STAGING, release, 1, ZIP_FILES[1])
            digest, more_counters = dataset.extract_file_data(
                root=STAGING,
                release=release,
                index=1,
                name=ZIP_FILES[1],
                filter=FILTER,
            )

            self.assertEqual(
                digest,
                "941a8be34ef1c5124b85182377d673b1fe7a9bf7c260fcb8995d9e620fa6a271",
            )

            self.assertListEqual(sorted(p.name for p in workdir.glob("*")), CSV_FILES)
            self.assertFileEqual(workdir / CSV_FILES[2], FIXTURE / "csv" / CSV_FILES[2])
            self.assertFileEqual(workdir / CSV_FILES[3], FIXTURE / "csv" / CSV_FILES[3])

            counters += more_counters
            self.assertEqual(counters["batch_count"], 2)
            self.assertEqual(counters["total_rows"], 100 + 102)
            self.assertEqual(counters["total_rows_with_keywords"], 12 + 1)
            self.assertEqual(counters["batch_rows"], 8 + 9)
            self.assertEqual(counters["batch_rows_with_keywords"], 2 + 0)

            memory = round(counters["batch_memory"] / 1_000)
            self.assertLess(10, memory)
            self.assertLess(memory, 30)

            self.assertTrue(batch1.exists())
            self.assertFileEqual(batch1, FIXTURE / release.batch_file(1))

        with self.subTest("analyze release data"):
            collector = Collector()
            release_metadata = pl.DataFrame(
                {
                    "batch_count": [2],
                    "total_rows": [665],
                    "total_rows_with_keywords": [212],
                }
            )
            dataset.analyze_release(
                STAGING, release.to_monthly(), release_metadata, collector
            )

            frame = collector.frame()
            frame_data = frame.to_dict(as_series=False)

            # import pprint

            # with open("data.txt", mode="w", encoding="utf8") as file:
            #     pprint.pprint(frame_data, stream=file)

            self.assertEqual(frame_data, EXPECTED_ANALYSIS)

            # Write to parquet
            frame.write_parquet(STAGING / Statistics.FILE)

        with self.subTest("check log file"):
            lines = LOGFILE.read_text("utf8").splitlines(keepends=True)

            offset = -1
            for offset, line in enumerate(lines):
                if 'staged file="sor-global-2024-03-14-full.zip"' in line:
                    break

            self.assertNotEqual(offset, -1)
            self.assertTrue(offset + 35 <= len(lines))
            self.assertIn("staged file", lines[offset + 0])
            self.assertIn("validated file", lines[offset + 1])
            self.assertIn('unarchived type="nested archive"', lines[offset + 2])
            self.assertIn('counted filter="none", rows=100', lines[offset + 3])
            self.assertIn('counted filter="with_keywords", rows=12', lines[offset + 4])
            self.assertIn("extracted rows=8", lines[offset + 5])
            self.assertIn('unarchived type="nested archive"', lines[offset + 6])
            self.assertIn('counted filter="none", rows=102', lines[offset + 7])
            self.assertIn('counted filter="with_keywords", rows=1', lines[offset + 8])
            # Trying to parse both CSV files in one Pola.rs operation fails:
            self.assertIn(
                'shantay︙WARNING︙failed to read CSV with strategy=1, using="globbing Pola.rs"',
                lines[offset + 9],
            )
            self.assertTrue(lines[offset + 10].startswith("Traceback"))
            self.assertTrue(lines[offset + 11].startswith("  File"))
            self.assertTrue(lines[offset + 12].startswith("    ).collect()"))
            self.assertTrue(lines[offset + 13].startswith("      ^^^^^^^"))
            self.assertTrue(lines[offset + 14].startswith("  File"))
            self.assertTrue(lines[offset + 15].startswith("    return function(*args, **kwargs)"))
            self.assertTrue(lines[offset + 16].startswith("           ^^^^^^^^^^^^^^^^^^^^^^^^^"))
            self.assertTrue(lines[offset + 17].startswith("  File"))
            self.assertTrue(lines[offset + 18].startswith("    return wrap_df(ldf.collect(engine, callback))"))
            self.assertTrue(lines[offset + 19].startswith("                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"))

            # Since the particulars of the traceback have changed over time,
            # make offsets relative to the next log line.
            offset = offset + 20

            self.assertTrue(
                lines[offset].startswith(
                    "polars.exceptions.ComputeError: could not parse"
                )
            )
            self.assertTrue(lines[offset + 1].startswith(""))
            self.assertTrue(
                lines[offset + 2].startswith(
                    "The current offset in the file is 131 bytes"
                )
            )
            self.assertTrue(lines[offset + 3].startswith(""))
            self.assertTrue(lines[offset + 4].startswith("You might want to try"))
            self.assertTrue(lines[offset + 5].startswith("- increasing"))
            self.assertTrue(lines[offset + 6].startswith("- specifying"))
            self.assertTrue(lines[offset + 7].startswith("- setting"))
            self.assertTrue(lines[offset + 8].startswith("- adding"))
            self.assertTrue(lines[offset + 9].startswith(""))
            self.assertTrue(
                lines[offset + 10].startswith("Original error: ```invalid csv file")
            )
            self.assertTrue(lines[offset + 11].startswith(""))
            self.assertTrue(lines[offset + 12].startswith('Field `"Napodobňovanie'))
            # Parsing the first CSV file by itself with Pola.rs works:
            self.assertIn(
                'extracted rows=8, strategy=2, using="Pola.rs"', lines[offset + 13]
            )
            # Parsing the second CSV file by itself with Pola.rs fails:
            self.assertIn(
                'failed to read CSV with strategy=2, using="Pola.rs"',
                lines[offset + 14],
            )
            # Parsing the second CSV fail by itself with Python's csv works:
            self.assertIn(
                'extracted rows=1, strategy=3, using="Python\'s CSV module"',
                lines[offset + 15],
            )


# Shantay's current analysis data format is a very long table...
EXPECTED_ANALYSIS = {
    "column": [
        "batch_count",
        "batch_rows",
        "batch_rows_with_keywords",
        "batch_memory",
        "total_rows",
        "total_rows_with_keywords",
        "rows",
        "decision_type",
        "decision_type",
        "decision_type",
        "decision_type",
        "decision_type",
        "decision_type",
        "decision_type",
        "decision_type",
        "decision_type",
        "decision_type",
        "decision_type",
        "decision_type",
        "decision_type",
        "decision_type",
        "decision_type",
        "decision_type",
        "decision_visibility",
        "decision_visibility",
        "decision_visibility",
        "decision_visibility",
        "visibility_restriction_duration",
        "decision_monetary",
        "monetary_restriction_duration",
        "decision_provision",
        "service_restriction_duration",
        "decision_account",
        "decision_account",
        "account_restriction_duration",
        "account_type",
        "decision_ground",
        "incompatible_content_illegal",
        "incompatible_content_illegal",
        "category",
        "category_addition",
        "category_addition",
        "category_addition",
        "category_addition",
        "category_specification",
        "category_specification",
        "category_specification",
        "category_specification",
        "category_specification",
        "category_specification",
        "category_specification",
        "content_type",
        "content_type",
        "content_type",
        "content_type",
        "content_type",
        "content_type",
        "content_type",
        "content_language",
        "moderation_delay",
        "disclosure_delay",
        "source_type",
        "source_type",
        "automated_detection",
        "automated_detection",
        "automated_decision",
        "automated_decision",
        "platform_name",
        "platform_name",
        "platform_name",
        "platform_name",
        "platform_name",
        "platform_name",
        "platform_name",
        "platform_name",
        "rows",
        "decision_type",
        "decision_type",
        "decision_type",
        "decision_type",
        "decision_type",
        "decision_type",
        "decision_type",
        "decision_type",
        "decision_type",
        "decision_type",
        "decision_type",
        "decision_type",
        "decision_type",
        "decision_type",
        "decision_type",
        "decision_type",
        "decision_visibility",
        "visibility_restriction_duration",
        "decision_monetary",
        "monetary_restriction_duration",
        "decision_provision",
        "service_restriction_duration",
        "decision_account",
        "account_restriction_duration",
        "account_type",
        "decision_ground",
        "incompatible_content_illegal",
        "category",
        "category_addition",
        "category_addition",
        "category_addition",
        "category_addition",
        "category_specification",
        "category_specification",
        "category_specification",
        "category_specification",
        "content_type",
        "content_type",
        "content_type",
        "content_type",
        "content_language",
        "moderation_delay",
        "disclosure_delay",
        "source_type",
        "automated_detection",
        "automated_decision",
        "platform_name",
        "platform_name",
    ],
    "count": [
        2,
        None,
        None,
        None,
        665,
        212,
        17,
        0,
        16,
        0,
        0,
        0,
        0,
        0,
        0,
        1,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        9,
        4,
        3,
        1,
        0,
        17,
        0,
        17,
        0,
        16,
        1,
        0,
        17,
        17,
        15,
        2,
        17,
        0,
        None,
        0,
        17,
        3,
        None,
        2,
        15,
        1,
        1,
        1,
        18,
        None,
        17,
        13,
        2,
        2,
        1,
        17,
        17,
        17,
        16,
        1,
        16,
        1,
        13,
        4,
        14,
        2,
        1,
        14,
        1,
        1,
        1,
        1,
        1,
        0,
        1,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        1,
        1,
        1,
        0,
        None,
        0,
        1,
        1,
        None,
        1,
        1,
        1,
        None,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
    ],
    "end_date": [
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
        dt.date(2024, 3, 31),
    ],
    "entity": [
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        "is_null",
        "vis",
        "mon",
        "vis_mon",
        "pro",
        "vis_pro",
        "mon_pro",
        "vis_mon_pro",
        "acc",
        "vis_acc",
        "mon_acc",
        "vis_mon_acc",
        "pro_acc",
        "vis_pro_acc",
        "mon_pro_acc",
        "vis_mon_pro_acc",
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        "elements",
        "elements_per_row",
        "rows_with_elements",
        None,
        "elements",
        "elements_per_row",
        "rows_with_elements",
        None,
        None,
        None,
        None,
        "elements",
        "elements_per_row",
        "rows_with_elements",
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        "with_category_specification",
        "with_category_specification",
        "with_category_specification",
        "with_category_specification",
        "with_category_specification",
        None,
        "is_null",
        "vis",
        "mon",
        "vis_mon",
        "pro",
        "vis_pro",
        "mon_pro",
        "vis_mon_pro",
        "acc",
        "vis_acc",
        "mon_acc",
        "vis_mon_acc",
        "pro_acc",
        "vis_pro_acc",
        "mon_pro_acc",
        "vis_mon_pro_acc",
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        "elements",
        "elements_per_row",
        "rows_with_elements",
        None,
        "elements",
        "elements_per_row",
        "rows_with_elements",
        None,
        "elements",
        "elements_per_row",
        "rows_with_elements",
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        "with_category_specification",
    ],
    "max": [
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        2,
        None,
        None,
        None,
        None,
        None,
        None,
        2,
        None,
        None,
        None,
        None,
        None,
        None,
        65059200000,
        172832000,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        1,
        None,
        None,
        None,
        1,
        None,
        None,
        None,
        0,
        95000,
        None,
        None,
        None,
        None,
        None,
    ],
    "mean": [
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        7206776470,
        91766588,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        0,
        95000,
        None,
        None,
        None,
        None,
        None,
    ],
    "min": [
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        0,
        95000,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        0,
        95000,
        None,
        None,
        None,
        None,
        None,
    ],
    "start_date": [
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
        dt.date(2024, 3, 1),
    ],
    "tag": [
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        "CSAM",
        "CSAM",
        "CSAM",
        "CSAM",
        "CSAM",
        "CSAM",
        "CSAM",
        "CSAM",
        "CSAM",
        "CSAM",
        "CSAM",
        "CSAM",
        "CSAM",
        "CSAM",
        "CSAM",
        "CSAM",
        "CSAM",
        "CSAM",
        "CSAM",
        "CSAM",
        "CSAM",
        "CSAM",
        "CSAM",
        "CSAM",
        "CSAM",
        "CSAM",
        "CSAM",
        "CSAM",
        "CSAM",
        "CSAM",
        "CSAM",
        "CSAM",
        "CSAM",
        "CSAM",
        "CSAM",
        "CSAM",
        "CSAM",
        "CSAM",
        "CSAM",
        "CSAM",
        "CSAM",
        "CSAM",
        "CSAM",
        "CSAM",
        "CSAM",
        "CSAM",
        "CSAM",
        "CSAM",
        "CSAM",
    ],
    "variant": [
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        "DECISION_VISIBILITY_OTHER",
        "DECISION_VISIBILITY_CONTENT_AGE_RESTRICTED",
        "DECISION_VISIBILITY_CONTENT_REMOVED",
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        "DECISION_ACCOUNT_SUSPENDED",
        None,
        None,
        "DECISION_GROUND_INCOMPATIBLE_CONTENT",
        None,
        "Yes",
        "STATEMENT_CATEGORY_PROTECTION_OF_MINORS",
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        "KEYWORD_CHILD_SEXUAL_ABUSE_MATERIAL",
        "KEYWORD_GROOMING_SEXUAL_ENTICEMENT_MINORS",
        "KEYWORD_AGE_SPECIFIC_RESTRICTIONS",
        None,
        None,
        None,
        "CONTENT_TYPE_VIDEO",
        "CONTENT_TYPE_IMAGE",
        "CONTENT_TYPE_TEXT",
        "CONTENT_TYPE_OTHER",
        None,
        None,
        None,
        "SOURCE_VOLUNTARY",
        "SOURCE_ARTICLE_16",
        "Yes",
        "No",
        "AUTOMATED_DECISION_FULLY",
        "AUTOMATED_DECISION_NOT_AUTOMATED",
        "TikTok",
        "Snapchat",
        "Google Shopping",
        "TikTok",
        "Snapchat",
        "Snapchat",
        "Snapchat",
        "Google Shopping",
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        "DECISION_VISIBILITY_CONTENT_REMOVED",
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        "DECISION_GROUND_INCOMPATIBLE_CONTENT",
        "Yes",
        "STATEMENT_CATEGORY_PROTECTION_OF_MINORS",
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        "KEYWORD_CHILD_SEXUAL_ABUSE_MATERIAL",
        None,
        None,
        None,
        "CONTENT_TYPE_IMAGE",
        None,
        None,
        None,
        "SOURCE_VOLUNTARY",
        "Yes",
        "AUTOMATED_DECISION_FULLY",
        "Snapchat",
        "Snapchat",
    ],
    "variant_too": [
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        "KEYWORD_AGE_SPECIFIC_RESTRICTIONS",
        "KEYWORD_GROOMING_SEXUAL_ENTICEMENT_MINORS",
        "KEYWORD_CHILD_SEXUAL_ABUSE_MATERIAL",
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        "KEYWORD_CHILD_SEXUAL_ABUSE_MATERIAL",
    ],
}
