#!/usr/bin/env python
#
# Copyright 2008 Zuza Software Foundation
#
# This file is part of the Translate Toolkit.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, see <http://www.gnu.org/licenses/>.
#
# moz-l10n-builder - takes a set of PO files, migrates them to a Mozilla build
# and creates XPIs and Windows .exe files.

"""Contains a Python-port of the moz-l10n-builder bash script."""

# NOTE: Because this script is adapted from a Bash script, some things in here
#       might be a little less Pythonic. See os.system() calls for more
#       details.

import glob
import os
import shutil
import sys
import tempfile
import time
from subprocess import PIPE, Popen


join = os.path.join

try:
    # Make sure that all convertion tools are available
    from translate.convert import moz2po, po2moz, po2prop, txt2po  # noqa: F401
except ImportError:
    raise Exception(
        "Could not find the Translate Toolkit convertion tools. Please check your installation."
    )

DEFAULT_TARGET_APP = "browser"
langpack_release = "1"
targetapp = "browser"
mozversion = "3"
l10ndir = "l10n"
mozilladir = "mozilla"
podir = "po"
podir_recover = podir + "-recover"
podir_updated = podir + "-updated"
potpacks = "potpacks"
popacks = "popacks"
#: Mapping of possible "targetapp"s to product names.
products = {"browser": "firefox"}

devnull = open(os.devnull, "wb")
#: Global program options
options = {"verbose": True}
USAGE = "Usage: %prog [options] <langs...|ALL>"


class CommandError(Exception):
    """Exception raised if a command does not return its expected value."""

    def __init__(self, cmd, status):
        self.cmd = cmd
        self.status = status

    def __str__(self):
        return '"%s" return unexptected status %d' % (self.cmd, self.status)


##### Utility Functions #####


def delfiles(pattern, path, files):
    """Delete files with names in C{files} matching glob-pattern C{glob} in the
    directory specified by C{path}.

    This function is meant to be used with C{os.walk}
    """
    path = os.path.abspath(path)
    match_files = glob.glob(join(path, pattern))
    for f in files:
        if join(path, f) in match_files:
            os.unlink(join(path, f))


def run(cmd, expected_status=0, stdout=None, stderr=None, shell=False):
    if options["verbose"]:
        print(">>> {} $ {}".format(os.getcwd(), " ".join(cmd)))
    p = Popen(cmd, stdout=stdout, stderr=stderr, shell=shell)
    cmd_status = p.wait()

    if stdout == PIPE:
        print(p.stdout.read())
    elif stderr == PIPE:
        print(p.stderr.read())

    if cmd_status != expected_status:
        print('!!! "%s" returned unexpected status %d' % (" ".join(cmd), cmd_status))
        # raise CommandError(cmd, cmd_status)


def get_langs(lang_args):
    """Returns the languages to handle based on the languages specified on the
    command-line.

    If "ALL" was specified, the languages are read from the Mozilla product's
    C{shipped-locales} file. If "ZA" was specified, all South African languages
    are selected.
    """

    langs = []

    if isinstance(lang_args, str):
        if lang_args == "ALL":
            lang_args = ["ALL"]
        elif lang_args == "ZA":
            lang_args = ["ZA"]
        else:
            lang_args = []

    if not lang_args:
        print(USAGE)
        sys.exit(1)

    for lang in lang_args:
        if lang == "ALL":
            # Get all available languages from the locales file
            locales_filename = join(mozilladir, targetapp, "locales", "shipped-locales")
            with open(locales_filename) as fh:
                for line in fh:
                    langcode = line.split()[0]
                    if langcode != "en-US":
                        langs.append(langcode)

        elif lang == "ZA":
            # South African languages
            langs = langs + [
                "af",
                "en_ZA",
                "nr",
                "nso",
                "ss",
                "st",
                "tn",
                "ts",
                "ve",
                "xh",
                "zu",
            ]
        elif lang != "en-US":
            langs.append(lang)

    langs = list(set(langs))  # Remove duplicates from langs

    print("Selected languages: %s" % (" ".join(langs)))

    return langs


#############################


def checkout(cvstag, langs):
    """Check-out needed files from Mozilla's CVS."""

    olddir = os.getcwd()
    if cvstag != "-A":
        cvstag = "-r %s" % (cvstag)

    if not os.path.exists(mozilladir):
        run(
            [
                "cvs",
                "-d:pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot",
                "co",
                cvstag,
                join(mozilladir, "client.mk"),
            ]
        )
        run(
            [
                "cvs",
                "-d:pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot",
                "co",
                join(mozilladir, "tools", "l10n"),
            ]
        )

    os.chdir(mozilladir)
    run(["cvs", "up", cvstag, "client.mk"])
    run(["make", "-f", "client.mk", "l10n-checkout", "MOZ_CO_PROJECT=%s" % (targetapp)])
    os.chdir(olddir)

    if not os.path.exists(l10ndir):
        run(
            [
                "cvs",
                "-d:pserver:anonymous@cvs-mirror.mozilla.org:/l10n",
                "co",
                "-d",
                l10ndir,
                "-l",
                "l10n",
            ]
        )

    os.chdir(l10ndir)
    for lang in langs:
        print("    %s" % (lang))
        buildlang = lang.replace("_", "-")
        if os.path.isdir(buildlang):
            run(["cvs", "up", buildlang])
        else:
            run(
                [
                    "cvs",
                    "-d:pserver:anonymous@cvs-mirror.mozilla.org:/l10n",
                    "co",
                    "-d",
                    buildlang,
                    join("l10n", buildlang),
                ]
            )
    os.chdir(olddir)

    # Make latest POT file
    for rmdir in ("en-US", "pot"):
        try:
            shutil.rmtree(join(l10ndir, rmdir))
        except OSError as oe:
            # "No such file or directory" errors are fine.
            # The rest we raise again.
            if oe.errno != 2:
                raise oe

    os.chdir(mozilladir)
    run(["cvs", "up", join("tools", "l10n")])
    run(
        [
            "python",
            "tools/l10n/l10n.py",
            "--dest=" + join(os.pardir, l10ndir),
            "--app=" + targetapp,
            "en-US",
        ]
    )
    os.chdir(olddir)

    os.chdir(l10ndir)
    run(["moz2po", "--progress=none", "-P", "--duplicates=msgctxt", "en-US", "pot"])

    # Delete the help-related POT-files, seeing as Firefox help is now on-line.
    try:
        shutil.rmtree(join("pot", "browser", "chrome", "help"))
    except OSError as oe:
        # "No such file or directory" errors are fine. The rest we raise again.
        if oe.errno != 2:
            raise oe

    if mozversion < "3":
        for f in [
            "en-US/browser/README.txt pot/browser/README.txt.pot",
            "en-US/browser/os2/README.txt pot/browser/os2/README.txt.pot",
            "en-US/mail/README.txt pot/mail/README.txt.pot",
            "en-US/mail/os2/README.txt pot/mail/os2/README.txt.pot",
        ]:
            run(["txt2po", "--progress=none", "-P", f])
    os.chdir(olddir)


def recover_lang(lang, buildlang):
    print("    %s" % (lang))
    if not os.path.isdir(join(podir_recover, buildlang)):
        os.makedirs(join(podir_recover, buildlang))

    run(
        [
            "moz2po",
            "--progress=none",
            "--errorlevel=traceback",
            "--duplicates=msgctxt",
            '--exclude=".#*"',
            "-t",
            join(l10ndir, "en-US"),
            join(l10ndir, buildlang),
            join(podir_recover, buildlang),
        ]
    )


def pack_pot(includes):
    timestamp = time.strftime("%Y%m%d")

    inc = []
    for fn in includes:
        if not os.path.exists(fn):
            print('!!! Warning: Path "%s" does not exist. Skipped.' % (fn))
        else:
            inc.append(fn)

    try:
        os.makedirs(potpacks)
    except OSError:
        pass

    packname = join(potpacks, f"{products[targetapp]}-{mozversion}-{timestamp}")
    run(
        [
            "tar",
            "cjf",
            packname + ".tar.bz2",
            join(l10ndir, "en-US"),
            join(l10ndir, "pot"),
        ]
        + inc
    )
    run(
        ["zip", "-qr9", packname + ".zip", join(l10ndir, "en-US"), join(l10ndir, "pot")]
        + inc
    )


def pack_po(lang, buildlang):
    timestamp = time.strftime("%Y%m%d")

    try:
        os.makedirs(popacks)
    except OSError:
        pass

    print("    %s" % (lang))
    packname = join(
        popacks,
        f"{products[targetapp]}-{mozversion}-{buildlang}-{timestamp}",
    )
    run(
        [
            "tar",
            "cjf",
            packname + ".tar.bz2",
            "--exclude",
            ".svn",
            join(l10ndir, buildlang),
            join(podir, buildlang),
        ]
    )
    run(
        [
            "zip",
            "-qr9",
            packname + ".zip",
            join(l10ndir, buildlang),
            join(podir, buildlang),
        ],
        "-x",
        "*.svn*",
    )


def pre_po2moz_hacks(lang, buildlang, debug):
    """Hacks that should be run before running C{po2moz}."""

    # Protect the real original PO dir
    temp_po = tempfile.mkdtemp()
    shutil.copytree(join(podir, buildlang), join(temp_po, buildlang))

    # Fix for languages that have no Windows codepage
    if lang == "ve":
        srcs = glob.glob(join(podir, "en_ZA", "browser", "installer", "*.properties"))
        dest = join(temp_po, buildlang, "browser", "installer")

        for src in srcs:
            shutil.copy2(src, dest)

    old = join(temp_po, buildlang)
    new = join(podir_updated, buildlang)
    templates = join(l10ndir, "pot")
    run(["pomigrate2", "--use-compendium", "--quiet", "--pot2po", old, new, templates])

    for dirpath, dirnames, filenames in os.walk(join(podir_updated, buildlang)):
        delfiles("*.html.po", dirpath, dirnames + filenames)
        delfiles("*.xhtml.po", dirpath, dirnames + filenames)

    if debug:
        olddir = os.getcwd()
        os.chdir(join("%s" % (podir_updated), buildlang))
        run(
            [
                "podebug",
                "--progress=none",
                "--errorlevel=traceback",
                "--ignore=mozilla",
                ".",
                ".",
            ]
        )
        os.chdir(olddir)

    # Create l10n related files
    if os.path.isdir(join(l10ndir, buildlang)):
        for dirpath, dirnames, filenames in os.walk(join(l10ndir, buildlang)):
            delfiles("*.dtd", dirpath, dirnames + filenames)
            delfiles("*.properties", dirpath, dirnames + filenames)

    shutil.rmtree(temp_po)


def post_po2moz_hacks(lang, buildlang):
    """Hacks that should be run after running C{po2moz}."""

    # Hack to fix creating Thunderber installer
    inst_inc_po = join(podir_updated, lang, "mail", "installer", "installer.inc.po")
    if os.path.isfile(inst_inc_po):
        tempdir = tempfile.mkdtemp()
        tmp_po = join(tempdir, "installer.%s.properties.po" % (lang))
        shutil.copy2(inst_inc_po, tmp_po)

        inst_inc = join(l10ndir, "en-US", "mail", "installer", "installer.inc")
        tmp_properties = join(tempdir, "installer.properties")
        shutil.copy2(inst_inc, tmp_properties)

        run(
            [
                "po2prop",
                "--progress=none",
                "--errorlevel=traceback",
                "-t",
                tmp_properties,  # -t /tmp/installer.properties
                tmp_po,  # /tmp/installer.$lang.properties.po
                tmp_po[:-3],
            ]
        )  # /tmp/installer.$lang.properties

        # mv /tmp/installer.$lang.properties \
        #    $l10ndir/$buildlang/mail/installer/installer.inc
        shutil.move(
            tmp_po[:-3], join(l10ndir, buildlang, "mail", "installer", "installer.inc")
        )

        shutil.rmtree(tempdir)

    def copyfile(filename, language):
        enUS = join(l10ndir, "en-US")
        dir, filename = os.path.split(filename)

        if dir.startswith(enUS):
            dir = dir[len(enUS) + 1 :]

        if os.path.isfile(join(enUS, dir, filename)):
            try:
                os.makedirs(join(l10ndir, language, dir))
            except OSError:
                pass  # Don't worry if the directory already exists
            shutil.copy2(join(enUS, dir, filename), join(l10ndir, language, dir))

    def copyfiletype(filetype, language):
        for dirpath, dirnames, filenames in os.walk(join(l10ndir, "en-US")):
            for f in dirnames + filenames:
                if f.endswith(filetype):
                    copyfile(join(dirpath, f), language)

    # Copy and update non-translatable files
    for ft in (".xhtml", ".html", ".rdf"):
        copyfiletype(ft, buildlang)

    for f in (
        join("browser", "extra-jar.mn"),
        join("browser", "firefox-l10n.js"),
        join("browser", "README.txt"),
        join("browser", "microsummary-generators", "list.txt"),
        join("browser", "profile", "chrome", "userChrome-example.css"),
        join("browser", "profile", "chrome", "userContent-example.css"),
        join("browser", "searchplugins", "list.txt"),
        join("extensions", "reporter", "chrome", "reporterOverlay.properties"),
        join("mail", "all-l10n.js"),
        join("toolkit", "chrome", "global", "intl.css"),
        join("toolkit", "installer", "windows", "charset.mk"),
    ):
        copyfile(f, buildlang)


def migrate_lang(lang, buildlang, recover, update_transl, debug):
    print("    %s" % (lang))

    if recover and not os.path.isdir(join(podir, buildlang)):
        # If we recovered the .po files for lang, but there is no other po
        # directory, we use the recovered .po files
        try:
            os.mkdir(podir)
        except OSError as oe:
            # "File exists" errors are fine. The rest we raise again.
            if oe.errno != 17:
                raise oe

        shutil.copytree(join(podir_recover, buildlang), join(podir, buildlang))

    if update_transl:
        olddir = os.getcwd()
        os.chdir(podir)
        run(["svn", "up", buildlang])
        os.chdir(olddir)

        os.chdir(l10ndir)
        run(["cvs", "up", buildlang])
        os.chdir(olddir)

    # Migrate language from current PO to latest POT
    if os.path.isdir(join(podir, ".svn")):
        shutil.rmtree(join(podir_updated, ".svn"))
        shutil.copytree(join(podir, ".svn"), podir_updated)
    if os.path.isdir(join(podir_updated, buildlang)):
        shutil.rmtree(join(podir_updated, buildlang))
    shutil.copytree(join(podir, buildlang), join(podir_updated, buildlang))

    for dirpath, dirnames, filenames in os.walk(join(podir_updated, buildlang)):
        delfiles("*.po", dirpath, dirnames + filenames)

    pre_po2moz_hacks(lang, buildlang, debug)

    ###################################################
    args = [
        "--progress=none",
        "--errorlevel=traceback",
        '--exclude=".svn"',
        "-t",
        join(l10ndir, "en-US"),
        "-i",
        join(podir_updated, buildlang),
        "-o",
        join(l10ndir, buildlang),
    ]

    if debug:
        args.append("--fuzzy")

    run(["po2moz"] + args)
    ###################################################

    post_po2moz_hacks(lang, buildlang)

    # Clean up where we made real tabs \t
    if mozversion < "3":
        run(
            [
                "sed",
                "-i",
                '"/^USAGE_MSG/s/\\\t/\t/g"',
                join(l10ndir, buildlang, "toolkit", "installer", "unix", "install.it"),
            ]
        )
        run(
            [
                "sed",
                "-i",
                '"/^#define MSG_USAGE/s/\\\t/\t/g"',
                join(l10ndir, buildlang, "browser", "installer", "installer.inc"),
            ]
        )

    # Fix bookmark file to point to the locale
    # FIXME - need some way to preserve this file if its been translated
    # already
    run(
        [
            "sed",
            "-i",
            "s/en-US/%s/g" % (buildlang),
            join(l10ndir, buildlang, "browser", "profile", "bookmarks.html"),
        ]
    )


def create_diff(lang, buildlang):
    """Create CVS-diffs for all languages."""

    if not os.path.isdir("diff"):
        os.mkdir("diff")

    print("    %s" % (lang))
    olddir = os.getcwd()

    os.chdir(l10ndir)
    outfile = join(os.pardir, "diff", buildlang + "-l10n.diff")
    with open(outfile, "w") as fh:
        run(["cvs", "diff", "--newfile", buildlang], stdout=fh)
    os.chdir(olddir)

    os.chdir(join(podir_updated, buildlang))
    outfile = join(os.pardir, os.pardir, "diff", buildlang + "-po.diff")
    with open(outfile, "w") as fh:
        run(
            [
                "svn",
                "diff",
                "--diff-cmd",
                r'diff -x "-u --ignore-matching-lines=^\"POT\|^\"X-Gene"',
            ],
            stdout=fh,
        )
    os.chdir(olddir)


def create_langpack(lang, buildlang):
    """Builds a XPI and installers for languages."""

    print("    %s" % (lang))

    olddir = os.getcwd()

    os.chdir(mozilladir)
    run(
        [
            "./configure",
            "--disable-compile-environment",
            "--disable-xft",
            "--enable-application=%s" % (targetapp),
        ]
    )
    os.chdir(olddir)

    os.chdir(join(mozilladir, targetapp, "locales"))
    langpack_name = "langpack-" + buildlang
    moz_brand_dir = join("other-licenses", "branding", "firefox")
    langpack_file = join(
        "'$(_ABS_DIST)'",
        "install",
        "Firefox-Languagepack-'$(MOZ_APP_VERSION)'-%s.'$(AB_CD)'.xpi"
        % langpack_release,
    )
    run(
        [
            "make",
            langpack_name,
            "MOZ_BRANDING_DIRECTORY=" + moz_brand_dir,
            "LANGPACK_FILE=" + langpack_file,
        ]
    )
    # The commented out (and very long) line below was found commented
    # out in the source script as well.
    # ( cd $mozilladir/$targetapp/locales; make repackage-win32-installer-af MOZ_BRANDING_DIRECTORY=other-licenses/branding/firefox WIN32_INSTALLER_IN=../../../Firefox-Setup-2.0.exe WIN32_INSTALLER_OUT='$(_ABS_DIST)'"/install/sea/Firefox-Setup-"'$(MOZ_APP_VERSION).$(AB_CD)'".exe" )
    os.chdir(olddir)


def create_option_parser():
    """Creates and returns cmd-line option parser."""

    from argparse import ArgumentParser

    parser = ArgumentParser(usage=USAGE)

    parser.add_argument(
        "-q",
        "--quiet",
        dest="verbose",
        action="store_false",
        default=True,
        help="Print as little as possible output.",
    )
    parser.add_argument(
        "--mozilla-product",
        dest="mozproduct",
        default=DEFAULT_TARGET_APP,
        help="Which product to build",
    )
    parser.add_argument(
        "--mozilla-checkout",
        dest="mozcheckout",
        action="store_true",
        default=False,
        help="Update of the Mozilla l10n files and POT files",
    )
    parser.add_argument(
        "--recover",
        dest="recover",
        action="store_true",
        default=False,
        help="build PO files from Mozilla's l10n files",
    )
    parser.add_argument(
        "--mozilla-tag",
        dest="moztag",
        default="-A",
        help="The tag to check out of CVS (implies --mozilla-checkout)",
    )
    parser.add_argument(
        "--update-translations",
        dest="update_translations",
        action="store_true",
        default=False,
        help="Update translations",
    )
    parser.add_argument(
        "--diff",
        dest="diff",
        action="store_true",
        default=False,
        help="Create diffs for migrated translations and localized Mozilla files",
    )
    parser.add_argument(
        "--potpack",
        dest="potpack",
        action="store_true",
        default=False,
        help="Create packages of the en-US and POT directories with today's timestamp",
    )
    parser.add_argument(
        "--pot-include",
        dest="potincl",
        action="append",
        default=[],
        help="Files to include in the POT pack (only used with --potpack)",
    )
    parser.add_argument(
        "--nomigrate",
        dest="migrate",
        action="store_false",
        default=True,
        help="Don't migrate",
    )
    parser.add_argument(
        "--popack",
        dest="popack",
        action="store_true",
        default=False,
        help="Create packages of all specified languages' PO-files with today's timestamp",
    )
    parser.add_argument(
        "--langpack",
        dest="langpack",
        action="store_true",
        default=False,
        help="Build a langpack",
    )
    parser.add_argument(
        "--debug",
        dest="debug",
        action="store_true",
        default=False,
        help="Add podebug debug markers",
    )

    return parser


def main(
    langs=["ALL"],
    mozproduct="browser",
    mozcheckout=False,
    moztag="-A",
    recover=False,
    potpack=False,
    potincl=[],
    migrate=True,
    popack=False,
    update_trans=False,
    debug=False,
    diff=False,
    langpack=False,
    verbose=True,
):
    global options, targetapp
    options["verbose"] = verbose
    targetapp = mozproduct
    langs = get_langs(langs)

    if mozcheckout:
        print("Checking out")
        checkout(moztag, langs)

    if potpack:
        print("Packing POT files")
        pack_pot(potincl)

    for lang in langs:
        buildlang = lang.replace("_", "-")

        if recover:
            print("Recovering")
            recover_lang(lang, buildlang)

        if migrate:
            print("Migrating")
            migrate_lang(lang, buildlang, recover, update_trans, debug)

        if popack:
            print("Creating PO-packs")
            pack_po(lang, buildlang)

        if diff:
            print("Creating diffs")
            create_diff(lang, buildlang)

        if langpack:
            print("Creating langpacks")
            create_langpack(lang, buildlang)

    print("FIN")
    devnull.close()


def main_cmd_line():
    args, langs = create_option_parser().parse_known_args()

    main(
        langs=langs,
        mozproduct=targetapp,
        mozcheckout=args.mozcheckout,
        moztag=args.moztag,
        recover=args.recover,
        potpack=args.potpack,
        potincl=args.potincl,
        migrate=args.migrate,
        popack=args.popack,
        update_trans=args.update_translations,
        debug=args.debug,
        diff=args.diff,
        langpack=args.langpack,
        verbose=args.verbose,
    )


if __name__ == "__main__":
    main_cmd_line()
