# chessdptdu.py
# Copyright 2008 Roger Marsh
# Licence: See LICENCE (BSD licence)

"""Chess database update using DPT single-step deferred update.

This module on Windows only.  Use multi-step module on Wine because Wine
support for a critical function used by single-step is not reliable. There
is no sure way to spot that module is running on Wine.

See www.dptoolkit.com for details of DPT

"""

import os
from io import StringIO
import multiprocessing  # Removed later by 'del multiprocessing'.

import tkinter
import tkinter.messagebox

# pylint will always give import-error message on non-Microsoft Windows
# systems.
# Wine counts as a Microsft Windows system.
# It is reasonable to not install 'dptdb.dptapi'.
# The importlib module is used to import chessdptdu if needed.
from dptdb.dptapi import FISTAT_DEFERRED_UPDATES

from solentware_base import dptdu_database
from solentware_base.core.constants import (
    FILEDESC,
    BRECPPG,
    TABLE_B_SIZE,
)

from ..core.filespec import (
    FileSpec,
    GAMES_FILE_DEF,
    PIECES_PER_POSITION,
    POSITIONS_PER_GAME,
)
from ..core.chessrecord import ChessDBrecordGameImport
from ..shared.archivedu import Archivedu

# Current practical way to determine if running on Wine, taking advantage of
# a problem found in ..core.uci which prevents UCI chess engines being used to
# do position analysis in this environment.  Here the consequence is Wine's
# useless answers to queries about memory usage made within DPT and the impact
# on the useful frequency of progress reports.
# CHUNKGAMES deals this this under Wine.  A safe value for a PC with 2Gb memory
# is 6144.  Almost get away with 8192 but it seems system housekeeping tasks,
# if nothing else, will cause this to fail at some point during a large import.
# Must be safe to add 8192 for every extra 2Gb, and possibly every extra 1Gb
# assuming system tasks are not greedy with the extra memory.
# The DPT segment size is 65280 because 32 bytes are reserved and 8160 bytes of
# the 8192 byte page are used for the bitmap.  Use 65536 as segment size and
# ignore the DPT segment size so report points are similar to other database
# engines.
# DB_SEGMENT_SIZE has no effect on processing apart from report points.
DB_SEGMENT_SIZE = 65536
try:
    multiprocessing.Queue()
    _DEFERRED_UPDATE_POINTS = (DB_SEGMENT_SIZE - 1,)
except OSError:
    _DEFERRED_UPDATE_POINTS = tuple(
        i
        for i in range(
            DB_SEGMENT_SIZE // 8 - 1, DB_SEGMENT_SIZE, DB_SEGMENT_SIZE // 8
        )
    )
del multiprocessing
del DB_SEGMENT_SIZE

CHUNKGAMES = 6144


class ChessdptduError(Exception):
    """Exception class for chessdptdu module."""


def chess_dptdu(
    dbpath,
    pgnpaths,
    file_records=None,
    reporter=lambda text, timestamp=True: None,
    estimated_number_of_games=0,
):
    """Open database, import games and close database."""
    cdb = ChessDatabase(dbpath, allowcreate=True)
    importer = ChessDBrecordGameImport()
    cdb.open_database(files=file_records)
    for pgnfile in pgnpaths:
        with open(pgnfile, "r", encoding="iso-8859-1") as source:
            importer.import_pgn(cdb, source, pgnfile, reporter=reporter)
    reporter("Finishing import: please wait.")
    reporter("", timestamp=False)
    cdb.close_database_contexts(files=file_records)
    cdb.open_database_contexts(files=file_records)
    status = True
    for file in (
        cdb.specification.keys() if file_records is None else file_records
    ):
        if (
            FISTAT_DEFERRED_UPDATES
            != cdb.table[file].get_file_parameters(cdb.dbenv)["FISTAT"][0]
        ):
            status = False
    cdb.close_database_contexts()
    return status


def chess_dptdu_chunks(
    dbpath,
    pgnpaths,
    file_records,
    reporter=lambda text, timestamp=True: None,
    estimated_number_of_games=0,
):
    """Open database, import games in fixed chunks and close database."""

    def write_chunk(pgnpath, sample=None):
        cdb.open_database(files=sample)

        # chess_dptdu_chunks does the reporting so do not pass reporter to
        # import_pgn, but the name of the source file is mandantory.
        importer.import_pgn(cdb, StringIO("".join(games)), pgnpath)

        cdb.close_database_contexts()
        games[:] = []

    cdb = ChessDatabase(dbpath, allowcreate=True)
    importer = ChessDBrecordGameImport()
    games = []
    gamelines = []
    newgamestring = '[Event "'
    records = file_records
    for pgnfile in pgnpaths:
        with open(pgnfile, "r", encoding="iso-8859-1") as inp:
            line = inp.readline()
            reporter("Extracting games from " + pgnfile)
            count = 0
            while line:
                if line.startswith(newgamestring):
                    games.append("".join(gamelines))
                    gamelines = []
                    if len(games) >= CHUNKGAMES:
                        write_chunk(pgnfile, sample=records)
                        count += 1
                        reporter(
                            " ".join(
                                (
                                    "Chunk",
                                    str(count),
                                    "written, total games added:",
                                    str(CHUNKGAMES * count),
                                )
                            )
                        )
                gamelines.append(line)
                line = inp.readline()
        if gamelines:
            games.append("".join(gamelines))
            write_chunk(pgnfile, sample=records)
        reporter("Extraction from " + pgnfile + " done")
        reporter("", timestamp=False)
    reporter("Finishing import: please wait.")
    reporter("", timestamp=False)
    cdb.open_database_contexts(files=file_records)
    status = True
    for file in file_records:
        if (
            FISTAT_DEFERRED_UPDATES
            != cdb.table[file].get_file_parameters(cdb.dbenv)["FISTAT"][0]
        ):
            status = False
    cdb.close_database_contexts()
    return status


class ChessDatabaseDeferred(dptdu_database.Database):
    """Provide deferred update methods for a database of games of chess.

    Subclasses must include a subclass of dptbase.Database as a superclass.

    """

    # ChessDatabaseDeferred.deferred_update_points is not needed in DPT, like
    # the similar attribute in chessdbbitdu.ChessDatabase for example, because
    # DPT does it's own memory management for deferred updates.
    # The same attribute is provided to allow the import_pgn method called in
    # this module's chess_dptdu and chess_dptdu_chunks functions to report
    # progress at regular intervals.
    # The values are set differently because Wine does not give a useful answer
    # to DPT's memory usage questions.
    deferred_update_points = frozenset(_DEFERRED_UPDATE_POINTS)

    def __init__(
        self,
        databasefolder,
        use_specification_items=None,
        dpt_records=None,
        **kargs,
    ):
        """Define chess database.

        **kargs
        allowcreate == False - remove file descriptions from FileSpec so
        that superclass cannot create them.
        Other arguments are passed through to superclass __init__.

        """
        ddnames = FileSpec(
            use_specification_items=use_specification_items,
            dpt_records=dpt_records,
        )
        # Deferred update for games file only
        for ddname in list(ddnames.keys()):
            if ddname != GAMES_FILE_DEF:
                del ddnames[ddname]

        if not kargs.get("allowcreate", False):
            try:
                for ddname in ddnames:
                    if FILEDESC in ddnames[ddname]:
                        del ddnames[ddname][FILEDESC]
            except Exception as error:
                if __name__ == "__main__":
                    raise
                raise ChessdptduError("DPT description invalid") from error

        try:
            super().__init__(ddnames, databasefolder, **kargs)
        except ChessdptduError as error:
            if __name__ == "__main__":
                raise
            raise ChessdptduError("DPT description invalid") from error

        # Retain import estimates for increase size by button actions
        self._import_estimates = None
        self._notional_record_counts = None
        # Methods passed by UI to populate report widgets
        self._reporter = None

    def open_database(self, files=None):
        """Delegate then raise ChessdptduError if database is inconsistent.

        Normally return None with the database open.

        Close the database and raise ChessdptduError exception if the
        database FISTAT parameter is not equal FISTAT_DEFERRED_UPDATES.

        """
        super().open_database(files=files)
        viewer = self.dbenv.Core().GetViewerResetter()
        for dbo in self.table.values():
            if (
                viewer.ViewAsInt("FISTAT", dbo.opencontext)
                != FISTAT_DEFERRED_UPDATES
            ):
                break
        else:
            if files is None:
                files = dict()
            self.increase_database_size(files=files)
            return
        self.close_database()
        raise ChessdptduError("A file is not in deferred update mode")

    def open_context_prepare_import(self):
        """Open all files normally."""
        super().open_database()

    def get_archive_names(self, files=()):
        """Return specified files and existing operating system files."""
        specs = {f for f in files if f in self.table}
        names = [v.file for k, v in self.table.items() if k in specs]
        exists = [
            os.path.basename(n)
            for n in names
            if os.path.exists(".".join((n, "bz2")))
        ]
        return (names, exists)

    def _get_pages_for_record_counts(self, counts=(0, 0)):
        """Return Table B and Table D pages needed for record counts."""
        brecppg = self.table[GAMES_FILE_DEF].filedesc[BRECPPG]
        return (
            counts[0] // brecppg,
            (counts[1] * self.table[GAMES_FILE_DEF].btod_factor) // brecppg,
        )

    def _get_database_table_sizes(self, files=None):
        """Return Table B and D size and usage in pages for files."""
        if files is None:
            files = dict()
        filesize = dict()
        for key, value in self.get_database_parameters(
            files=list(files.keys())
        ).items():
            filesize[key] = (
                value["BSIZE"],
                value["BHIGHPG"],
                value["DSIZE"],
                value["DPGSUSED"],
            )
        increase = self.get_database_increase(files=files)
        self.close_database_contexts()
        return filesize, increase

    def add_import_buttons(
        self, master, try_command_wrapper, try_event_wrapper, bind, widget
    ):
        """Add button actions for DPT to Import dialogue.

        Increase data and index space available.

        """
        index = tkinter.Button(
            master=master,
            text="Increase Index",
            underline=13,
            command=try_command_wrapper(self._increase_index, master),
        )
        index.pack(side=tkinter.RIGHT, padx=12)
        bind(widget, "<Alt-x>", try_event_wrapper(self._increase_index))
        data = tkinter.Button(
            master=master,
            text="Increase Data",
            underline=9,
            command=try_command_wrapper(self._increase_data, master),
        )
        data.pack(side=tkinter.RIGHT, padx=12)
        bind(widget, "<Alt-d>", try_event_wrapper(self._increase_data))

    def _increase_data(self, event=None):
        """Add maximum of current free space and default size to Table B.

        event is ignored and is present for compatibility between button click
        and keypress,

        """
        del event
        self.open_database_contexts(files=(GAMES_FILE_DEF,))
        increase_done = False
        for key, value in self.get_database_parameters(
            files=(GAMES_FILE_DEF,)
        ).items():
            bsize = value["BSIZE"]
            bused = max(0, value["BHIGHPG"])
            bneeded = self._get_pages_for_record_counts(
                self._notional_record_counts[key]
            )[0]
            bincrease = min(bneeded * 2, bsize - bused)
            message = "".join(
                (
                    "The free data size of the ",
                    key,
                    " file will be increased from ",
                    str(bsize - bused),
                    " pages to ",
                    str(bincrease + bsize - bused),
                    " pages.",
                )
            )
            if len(self.table[key].get_extents()) % 2 == 0:
                message = "".join(
                    (
                        message,
                        "\n\nAt present it is better to do index increases ",
                        "first for this file, if you need to do any, because ",
                        "a new extent (fragment) would not be needed.",
                    )
                )
            if tkinter.messagebox.askyesno(
                title="Increase Data Size",
                message="".join(
                    (
                        message,
                        "\n\nDo you want to increase the data size?",
                    )
                ),
            ):
                increase_done = True
                self.table[key].opencontext.Increase(bincrease, False)
        if increase_done:
            self._reporter.append_text(
                " ".join(
                    (
                        "Recalculation of planned database size increases",
                        "after data size increase by user action.",
                    )
                )
            )
            self._reporter.append_text_only("")
            self._report_plans_for_estimate()
        self.close_database_contexts()

    def _increase_index(self, event=None):
        """Add maximum of current free space and default size to Table D.

        event is ignored and is present for compatibility between button click
        and keypress,

        """
        del event
        self.open_database_contexts(files=(GAMES_FILE_DEF,))
        increase_done = False
        for key, value in self.get_database_parameters(
            files=(GAMES_FILE_DEF,)
        ).items():
            dsize = value["DSIZE"]
            dused = value["DPGSUSED"]
            dneeded = self._get_pages_for_record_counts(
                self._notional_record_counts[key]
            )[1]
            dincrease = min(dneeded * 2, dsize - dused)
            message = "".join(
                (
                    "The free index size of the ",
                    key,
                    " file will be increased from ",
                    str(dsize - dused),
                    " pages to ",
                    str(dincrease + dsize - dused),
                    " pages.",
                )
            )
            if len(self.table[key].get_extents()) % 2 != 0:
                message = "".join(
                    (
                        message,
                        "\n\nAt present it is better to do data increases ",
                        "first for this file, if you need to do any, because ",
                        "a new extent (fragment) would not be needed.",
                    )
                )
            if tkinter.messagebox.askyesno(
                title="Increase Index Size",
                message="".join(
                    (
                        message,
                        "\n\nDo you want to increase the index size?",
                    )
                ),
            ):
                increase_done = True
                self.table[key].opencontext.Increase(dincrease, True)
        if increase_done:
            self._reporter.append_text(
                " ".join(
                    (
                        "Recalculation of planned database size increases",
                        "after index size increase by user action.",
                    )
                )
            )
            self._reporter.append_text_only("")
            self._report_plans_for_estimate()
        self.close_database_contexts()

    def get_file_sizes(self):
        """Return dictionary of notional record counts for data and index."""
        return self._notional_record_counts

    def report_plans_for_estimate(self, estimates, reporter):
        """Calculate and report file size adjustments to do import.

        Note the reporter and headline methods for initial report and possible
        later recalculations.

        Pass estimates through to self._report_plans_for_estimate

        """
        # See comment near end of class definition Chess in relative module
        # ..gui.chess for explanation of this change.
        self._reporter = reporter
        self._report_plans_for_estimate(estimates=estimates)

    def _report_plans_for_estimate(self, estimates=None):
        """Recalculate and report file size adjustments to do import.

        Create dictionary of effective game counts for sizing Games file.
        This will be passed to the import job which will increase Table B and
        Table D according to file specification.

        The counts for Table B and Table D can be different.  If the average
        data bytes per game is greater than Page size / Records per page the
        count must be increased to allow for the unused record numbers.  If
        the average positions per game or pieces per position are not the
        values used to calculate the steady-state ratio of Table B to Table D
        the count must be adjusted to compensate.

        """
        append_text = self._reporter.append_text
        append_text_only = self._reporter.append_text_only
        if estimates is not None:
            self._import_estimates = estimates
        (
            gamecount,
            bytes_per_game,
            positions_per_game,
            pieces_per_game,
        ) = self._import_estimates[:4]
        brecppg = self.table[GAMES_FILE_DEF].filedesc[BRECPPG]
        d_count = (gamecount * (positions_per_game + pieces_per_game)) // (
            POSITIONS_PER_GAME * (1 + PIECES_PER_POSITION)
        )
        if bytes_per_game > (TABLE_B_SIZE // brecppg):
            b_count = int(
                (gamecount * bytes_per_game) / (TABLE_B_SIZE / brecppg)
            )
        else:
            b_count = gamecount
        self._notional_record_counts = {
            GAMES_FILE_DEF: (b_count, d_count),
        }
        append_text("Current file size and free space:")
        free = dict()
        sizes, increases = self._get_database_table_sizes(
            files=self._notional_record_counts
        )
        for filename, bdsize in sizes.items():
            bsize, bused, dsize, dused = bdsize
            bused = max(0, bused)
            free[filename] = (bsize - bused, dsize - dused)
            append_text_only(filename)
            append_text_only(
                " ".join(("Current data area size", str(bsize), "pages"))
            )
            append_text_only(
                " ".join(("Current index area size", str(dsize), "pages"))
            )
            append_text_only(
                " ".join(
                    ("Current data area free", str(bsize - bused), "pages")
                )
            )
            append_text_only(
                " ".join(
                    ("Current index area free", str(dsize - dused), "pages")
                )
            )
        append_text_only("")
        append_text("File space needed for import:")
        for filename, nr_count in self._notional_record_counts.items():
            append_text_only(filename)
            b_pages, d_pages = self._get_pages_for_record_counts(nr_count)
            append_text_only(
                " ".join(("Estimated", str(b_pages), "pages needed for data"))
            )
            append_text_only(
                " ".join(
                    ("Estimated", str(d_pages), "pages needed for indexes")
                )
            )
        append_text_only("")
        append_text("File size increases planned and free space when done:")
        for filename, increments in increases.items():
            b_incr, d_incr = increments
            b_free, d_free = free[filename]
            append_text_only(filename)
            append_text_only(
                " ".join(("Data area increase", str(b_incr), "pages"))
            )
            append_text_only(
                " ".join(("Index area increase", str(d_incr), "pages"))
            )
            append_text_only(
                " ".join(("Data area free", str(b_incr + b_free), "pages"))
            )
            append_text_only(
                " ".join(("Index area free", str(d_incr + d_free), "pages"))
            )
        append_text_only("")
        append_text_only(
            "".join(
                (
                    "Comparison of the required and free data or index ",
                    "space may justify using the Increase Data and, or, ",
                    "Increase Index actions to get more space immediately ",
                    "given your knowledge of the PGN file being imported.",
                )
            )
        )
        append_text_only("")
        append_text_only("")


class ChessDatabase(Archivedu, ChessDatabaseDeferred):
    """Provide single-step deferred update for a database of games of chess."""


if __name__ == "__main__":

    import tkinter.ttk
    import tkinter.filedialog

    from solentware_bind.gui.bindings import Bindings

    binder = Bindings()
    root = tkinter.Tk()
    root.wm_title("Sample do_deferred_updates")
    frame = tkinter.ttk.Frame(master=root)
    text = tkinter.Text(master=frame, wrap=tkinter.WORD)
    frame.pack(fill=tkinter.Y, expand=tkinter.TRUE)
    text.pack(fill=tkinter.BOTH, expand=tkinter.TRUE)
    databasepath = tkinter.filedialog.askdirectory(
        parent=root, title="Select database directory", initialdir="~"
    )
    if databasepath:
        text.insert(tkinter.END, "Database directory: " + databasepath + "\n")
    else:
        text.insert(tkinter.END, "No database selected\n")

    def quit_(*a):
        """Destroy application."""
        del a
        root.destroy()

    def show_menu(event=None):
        """Post popup menu at pointer location."""
        menu.tk_popup(*event.widget.winfo_pointerxy())

    menu = tkinter.Menu(master=root, tearoff=False)
    if databasepath:

        class _FilePath:
            """Wrap filepath as a class attribute.

            pylint by default reports problems with filepath as a module
            constant and as a global in the proceed() method.

            Class attributes are allowed any format by pylint by default.
            """

            filepath = None

        def do_function(dbp, pgnpath):
            """Run chess_dptdu with database at dbp and PGN file at fp."""
            chess_dptdu(dbp, pgnpath)
            text.insert(tkinter.END, "Process finished\nProceed or Quit?\n\n")

        def proceed(*a):
            """Run a process by the do_function function.

            *a soaks up any arguments: in particular event if proceed is
            invoked by KeyPress rather than a menu command.

            """
            del a
            filepath = _FilePath.filepath  # Used to be 'global filepath'.
            if filepath:
                text.insert(tkinter.END, "\nProcess started, please wait.\n")
                text.after(1, do_function, *(databasepath, [filepath]))
                _FilePath.filepath = None
            elif databasepath:
                filepath = tkinter.filedialog.askopenfilename(
                    parent=root, title="Select data file", initialdir="~"
                )
                _FilePath.filepath = filepath
                if filepath:
                    text.insert(tkinter.END, "Data file: " + filepath + "\n")
                else:
                    text.insert(tkinter.END, "No data file selected\n")

        menu.add_separator()
        menu.add_command(
            label="Proceed", command=proceed, accelerator="Alt F2"
        )
        menu.add_command(label="Quit", command=quit_, accelerator="Alt F11")
        menu.add_separator()
        text.insert(
            tkinter.END, "\nRight-click to proceed after each step\n\n"
        )
        binder.bind(text, "<Alt-KeyPress-F2>", function=proceed)
        binder.bind(text, "<Alt-KeyPress-F11>", function=quit_)
        binder.bind(text, "<ButtonPress-3>", function=show_menu)
    else:
        menu.add_separator()
        menu.add_command(label="Quit", command=quit_, accelerator="Alt F11")
        menu.add_separator()
        text.insert(tkinter.END, "\nRight-click to quit\n")
        binder.bind(text, "<Alt-KeyPress-F11>", function=quit_)
        binder.bind(text, "<ButtonPress-3>", function=show_menu)
    root.mainloop()
