#
# ====================================================================
#    Licensed to the Apache Software Foundation (ASF) under one
#    or more contributor license agreements.  See the NOTICE file
#    distributed with this work for additional information
#    regarding copyright ownership.  The ASF licenses this file
#    to you under the Apache License, Version 2.0 (the
#    "License"); you may not use this file except in compliance
#    with the License.  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing,
#    software distributed under the License is distributed on an
#    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
#    KIND, either express or implied.  See the License for the
#    specific language governing permissions and limitations
#    under the License.
######################################################################

import sys     # for argv[]
import os
import shutil  # for rmtree()
import re
import stat    # for ST_MODE
import subprocess
import copy    # for deepcopy()
import time    # for time()
import traceback # for print_exc()
import threading
import optparse # for argument parsing

try:
    # Python >=3.0
    import queue
    from urllib.parse import quote as urllib_parse_quote
    from urllib.parse import unquote as urllib_parse_unquote
except ImportError:
    # Python <3.0
    import Queue as queue
    from urllib import quote as urllib_parse_quote
    from urllib import unquote as urllib_parse_unquote

import cvs2hg_test
from cvs2hg_test import Failure
from cvs2hg_test import Skip


######################################################################
#
#  HOW TO USE THIS MODULE:
#
#  Write a new python script that
#
#     1) imports this 'cvs2hg_test' package
#
#     2) contains a number of related 'test' routines.  (Each test
#        routine should take no arguments, and return None on success
#        or throw a Failure exception on failure.  Each test should
#        also contain a short docstring.)
#
#     3) places all the tests into a list that begins with None.
#
#     4) calls cvs2hg_test.main.client_test() on the list.
#
#  Also, your tests will probably want to use some of the common
#  routines in the 'Utilities' section below.
#
#####################################################################
# Global stuff

class SVNProcessTerminatedBySignal(Failure):
    "Exception raised if a spawned process segfaulted, aborted, etc."
    pass

class SVNLineUnequal(Failure):
    "Exception raised if two lines are unequal"
    pass

class SVNUnmatchedError(Failure):
    "Exception raised if an expected error is not found"
    pass

class SVNCommitFailure(Failure):
    "Exception raised if a commit failed"
    pass

class SVNRepositoryCopyFailure(Failure):
    "Exception raised if unable to copy a repository"
    pass

class SVNRepositoryCreateFailure(Failure):
    "Exception raised if unable to create a repository"
    pass

# Windows specifics
if sys.platform == 'win32':
    windows = True
    file_scheme_prefix = 'file:///'
    _exe = '.exe'
    _bat = '.bat'
    os.environ['SVN_DBG_STACKTRACES_TO_STDERR'] = 'y'
else:
    windows = False
    file_scheme_prefix = 'file://'
    _exe = ''
    _bat = ''

# The location of our mock svneditor script.
if windows:
    svneditor_script = os.path.join(sys.path[0], 'svneditor.bat')
else:
    svneditor_script = os.path.join(sys.path[0], 'svneditor.py')

# Username and password used by the working copies
wc_author = 'jrandom'
wc_passwd = 'rayjandom'

# Username and password used by the working copies for "second user"
# scenarios
wc_author2 = 'jconstant' # use the same password as wc_author

# Set C locale for command line programs
os.environ['LC_ALL'] = 'C'

# This function mimics the Python 2.3 urllib function of the same name.
def pathname2url(path):
    """Convert the pathname PATH from the local syntax for a path to the form
    used in the path component of a URL. This does not produce a complete URL.
    The return value will already be quoted using the quote() function."""

    # Don't leave ':' in file://C%3A/ escaped as our canonicalization
    # rules will replace this with a ':' on input.
    return urllib_parse_quote(path.replace('\\', '/')).replace('%3A', ':')

# This function mimics the Python 2.3 urllib function of the same name.
def url2pathname(path):
    """Convert the path component PATH from an encoded URL to the local syntax
    for a path. This does not accept a complete URL. This function uses
    unquote() to decode PATH."""
    return os.path.normpath(urllib_parse_unquote(path))

######################################################################
# The locations of the svn, svnadmin and svnlook binaries, relative to
# the only scripts that import this file right now (they live in ../).
# Use --bin to override these defaults.
hg_binary = "hg"

# Location to the pristine repository, will be calculated from test_area_url
# when we know what the user specified for --url.
# pristine_url = None

# Global variable to track all of our options
options = None

# End of command-line-set global variables.
######################################################################

# All temporary repositories and working copies are created underneath
# this dir, so there's one point at which to mount, e.g., a ramdisk.
work_dir = "test-temporary-work-dir"

# Constant for the merge info property.
SVN_PROP_MERGEINFO = "svn:mergeinfo"

# Where we want all the repositories and working copies to live.
# Each test will have its own!
general_repo_dir = os.path.join(work_dir, "repositories")
general_wc_dir = os.path.join(work_dir, "working_copies")

# temp directory in which we will create our 'pristine' local
# repository and other scratch data.  This should be removed when we
# quit and when we startup.
temp_dir = os.path.join(work_dir, 'local_tmp')

# (derivatives of the tmp dir.)
pristine_dir = os.path.join(temp_dir, "repos")
greek_dump_dir = os.path.join(temp_dir, "greekfiles")
default_config_dir = os.path.abspath(os.path.join(temp_dir, "config"))

#
# Our pristine greek-tree state.
#
# If a test wishes to create an "expected" working-copy tree, it should
# call main.greek_state.copy().  That method will return a copy of this
# State object which can then be edited.
#
_item = cvs2hg_test.wc.StateItem
greek_state = cvs2hg_test.wc.State('', {
  'iota'        : _item("This is the file 'iota'.\n"),
  'A'           : _item(),
  'A/mu'        : _item("This is the file 'mu'.\n"),
  'A/B'         : _item(),
  'A/B/lambda'  : _item("This is the file 'lambda'.\n"),
  'A/B/E'       : _item(),
  'A/B/E/alpha' : _item("This is the file 'alpha'.\n"),
  'A/B/E/beta'  : _item("This is the file 'beta'.\n"),
  'A/B/F'       : _item(),
  'A/C'         : _item(),
  'A/D'         : _item(),
  'A/D/gamma'   : _item("This is the file 'gamma'.\n"),
  'A/D/G'       : _item(),
  'A/D/G/pi'    : _item("This is the file 'pi'.\n"),
  'A/D/G/rho'   : _item("This is the file 'rho'.\n"),
  'A/D/G/tau'   : _item("This is the file 'tau'.\n"),
  'A/D/H'       : _item(),
  'A/D/H/chi'   : _item("This is the file 'chi'.\n"),
  'A/D/H/psi'   : _item("This is the file 'psi'.\n"),
  'A/D/H/omega' : _item("This is the file 'omega'.\n"),
  })


######################################################################
# Utilities shared by the tests
def wrap_ex(func):
    "Wrap a function, catch, print and ignore exceptions"
    def w(*args, **kwds):
        try:
            return func(*args, **kwds)
        except Failure as ex:
            if ex.__class__ != Failure or ex.args:
                ex_args = str(ex)
                if ex_args:
                    print('EXCEPTION: %s: %s' % (ex.__class__.__name__, ex_args))
                else:
                    print('EXCEPTION: %s' % ex.__class__.__name__)
    return w

def setup_development_mode():
    "Wraps functions in module actions"
    l = [ 'run_and_verify_svn',
          'run_and_verify_svnversion',
          'run_and_verify_load',
          'run_and_verify_dump',
          'run_and_verify_checkout',
          'run_and_verify_export',
          'run_and_verify_update',
          'run_and_verify_merge',
          'run_and_verify_switch',
          'run_and_verify_commit',
          'run_and_verify_unquiet_status',
          'run_and_verify_status',
          'run_and_verify_diff_summarize',
          'run_and_verify_diff_summarize_xml',
          'run_and_validate_lock']

    for func in l:
        setattr(cvs2hg_test.actions, func, wrap_ex(getattr(cvs2hg_test.actions, func)))

def get_admin_name():
    "Return name of SVN administrative subdirectory."

    if (windows or sys.platform == 'cygwin') \
        and 'SVN_ASP_DOT_NET_HACK' in os.environ:
        return '_svn'
    else:
        return '.svn'

def get_start_commit_hook_path(repo_dir):
    "Return the path of the start-commit-hook conf file in REPO_DIR."

    return os.path.join(repo_dir, "hooks", "start-commit")


def get_pre_commit_hook_path(repo_dir):
    "Return the path of the pre-commit-hook conf file in REPO_DIR."

    return os.path.join(repo_dir, "hooks", "pre-commit")


def get_post_commit_hook_path(repo_dir):
    "Return the path of the post-commit-hook conf file in REPO_DIR."

    return os.path.join(repo_dir, "hooks", "post-commit")

def get_pre_revprop_change_hook_path(repo_dir):
    "Return the path of the pre-revprop-change hook script in REPO_DIR."

    return os.path.join(repo_dir, "hooks", "pre-revprop-change")

def get_svnserve_conf_file_path(repo_dir):
    "Return the path of the svnserve.conf file in REPO_DIR."

    return os.path.join(repo_dir, "conf", "svnserve.conf")

def get_fsfs_conf_file_path(repo_dir):
    "Return the path of the fsfs.conf file in REPO_DIR."

    return os.path.join(repo_dir, "db", "fsfs.conf")

def get_fsfs_format_file_path(repo_dir):
    "Return the path of the format file in REPO_DIR."

    return os.path.join(repo_dir, "db", "format")

# Run any binary, logging the command line and return code
def run_command(command, error_expected, binary_mode=0, *varargs):
    """Run COMMAND with VARARGS. Return exit code as int; stdout, stderr
    as lists of lines (including line terminators).  See run_command_stdin()
    for details.  If ERROR_EXPECTED is None, any stderr also will be printed."""

    return run_command_stdin(command, error_expected, 0, binary_mode,
                             None, *varargs)

# A regular expression that matches arguments that are trivially safe
# to pass on a command line without quoting on any supported operating
# system:
_safe_arg_re = re.compile(r'^[A-Za-z\d\.\_\/\-\:\@]+$')

def _quote_arg(arg):
    """Quote ARG for a command line.

    Simply surround every argument in double-quotes unless it contains
    only universally harmless characters.

    WARNING: This function cannot handle arbitrary command-line
    arguments.  It can easily be confused by shell metacharacters.  A
    perfect job would be difficult and OS-dependent (see, for example,
    http://msdn.microsoft.com/library/en-us/vccelng/htm/progs_12.asp).
    In other words, this function is just good enough for what we need
    here."""

    arg = str(arg)
    if _safe_arg_re.match(arg):
        return arg
    else:
        if os.name != 'nt':
            arg = arg.replace('$', '\$')
        return '"%s"' % (arg,)

def open_pipe(command, bufsize=0, stdin=None, stdout=None, stderr=None):
    """Opens a subprocess.Popen pipe to COMMAND using STDIN,
    STDOUT, and STDERR.  BUFSIZE is passed to subprocess.Popen's
    argument of the same name.

    Returns (infile, outfile, errfile, waiter); waiter
    should be passed to wait_on_pipe."""
    command = [str(x) for x in command]

    # On Windows subprocess.Popen() won't accept a Python script as
    # a valid program to execute, rather it wants the Python executable.
    if (sys.platform == 'win32') and (command[0].endswith('.py')):
        command.insert(0, sys.executable)

    # Quote only the arguments on Windows.  Later versions of subprocess,
    # 2.5.2+ confirmed, don't require this quoting, but versions < 2.4.3 do.
    if sys.platform == 'win32':
        args = command[1:]
        args = ' '.join([_quote_arg(x) for x in args])
        command = command[0] + ' ' + args
        command_string = command
    else:
        command_string = ' '.join(command)

    if not stdin:
        stdin = subprocess.PIPE
    if not stdout:
        stdout = subprocess.PIPE
    if not stderr:
        stderr = subprocess.PIPE

    p = subprocess.Popen(command,
                         bufsize,
                         stdin=stdin,
                         stdout=stdout,
                         stderr=stderr,
                         close_fds=not windows)
    return p.stdin, p.stdout, p.stderr, (p, command_string)

def wait_on_pipe(waiter, binary_mode, stdin=None):
    """WAITER is (KID, COMMAND_STRING).  Wait for KID (opened with open_pipe)
    to finish, dying if it does.  If KID fails, create an error message
    containing any stdout and stderr from the kid.  Show COMMAND_STRING in
    diagnostic messages.  Normalize Windows line endings of stdout and stderr
    if not BINARY_MODE.  Return KID's exit code as int; stdout, stderr as
    lists of lines (including line terminators)."""
    if waiter is None:
        return

    kid, command_string = waiter
    stdout, stderr = kid.communicate(stdin)
    exit_code = kid.returncode

    # Normalize Windows line endings if in text mode.
    if windows and not binary_mode:
        stdout = stdout.replace('\r\n', '\n')
        stderr = stderr.replace('\r\n', '\n')

    # Convert output strings to lists.
    stdout_lines = stdout.splitlines(True)
    stderr_lines = stderr.splitlines(True)

    if exit_code < 0:
        if not windows:
            exit_signal = os.WTERMSIG(-exit_code)
        else:
            exit_signal = exit_code

        if stdout_lines is not None:
            sys.stdout.write("".join(stdout_lines))
            sys.stdout.flush()
        if stderr_lines is not None:
            sys.stderr.write("".join(stderr_lines))
            sys.stderr.flush()
        if options.verbose:
            # show the whole path to make it easier to start a debugger
            sys.stderr.write("CMD: %s terminated by signal %d\n"
                             % (command_string, exit_signal))
            sys.stderr.flush()
        raise SVNProcessTerminatedBySignal
    else:
        if exit_code and options.verbose:
            sys.stderr.write("CMD: %s exited with %d\n"
                             % (command_string, exit_code))
            sys.stderr.flush()
        return stdout_lines, stderr_lines, exit_code

def spawn_process(command, bufsize=0, binary_mode=0, stdin_lines=None,
                  *varargs):
    """Run any binary, supplying input text, logging the command line.
    BUFSIZE dictates the pipe buffer size used in communication with the
    subprocess: 0 means unbuffered, 1 means line buffered, any other
    positive value means use a buffer of (approximately) that size.
    A negative bufsize means to use the system default, which usually
    means fully buffered. The default value for bufsize is 0 (unbuffered).
    Normalize Windows line endings of stdout and stderr if not BINARY_MODE.
    Return exit code as int; stdout, stderr as lists of lines (including
    line terminators)."""
    if stdin_lines and not isinstance(stdin_lines, list):
        raise TypeError("stdin_lines should have list type")

    # Log the command line
    if options.verbose and not command.endswith('.py'):
        sys.stdout.write('CMD: %s %s\n' % (os.path.basename(command),
                                          ' '.join([_quote_arg(x) for x in varargs])))
        sys.stdout.flush()

    infile, outfile, errfile, kid = open_pipe([command] + list(varargs), bufsize)

    if stdin_lines:
        for x in stdin_lines:
            infile.write(x)

    stdout_lines, stderr_lines, exit_code = wait_on_pipe(kid, binary_mode)
    infile.close()

    outfile.close()
    errfile.close()

    return exit_code, stdout_lines, stderr_lines

def run_command_stdin(command, error_expected, bufsize=0, binary_mode=0,
                      stdin_lines=None, *varargs):
    """Run COMMAND with VARARGS; input STDIN_LINES (a list of strings
    which should include newline characters) to program via stdin - this
    should not be very large, as if the program outputs more than the OS
    is willing to buffer, this will deadlock, with both Python and
    COMMAND waiting to write to each other for ever.  For tests where this
    is a problem, setting BUFSIZE to a sufficiently large value will prevent
    the deadlock, see spawn_process().
    Normalize Windows line endings of stdout and stderr if not BINARY_MODE.
    Return exit code as int; stdout, stderr as lists of lines (including
    line terminators).
    If ERROR_EXPECTED is None, any stderr also will be printed."""

    if options.verbose:
        start = time.time()

    exit_code, stdout_lines, stderr_lines = spawn_process(command,
                                                          bufsize,
                                                          binary_mode,
                                                          stdin_lines,
                                                          *varargs)

    if options.verbose:
        stop = time.time()
        print('<TIME = %.6f>' % (stop - start))
        for x in stdout_lines:
            sys.stdout.write(x)
        for x in stderr_lines:
            sys.stdout.write(x)

    if (not error_expected) and (stderr_lines):
        if not options.verbose:
            for x in stderr_lines:
                sys.stdout.write(x)
        raise Failure

    return exit_code, \
           [line for line in stdout_lines if not line.startswith("DBG:")], \
           stderr_lines

def create_config_dir(cfgdir, config_contents=None, server_contents=None):
    "Create config directories and files"

    # config file names
    cfgfile_cfg = os.path.join(cfgdir, 'config')
    cfgfile_srv = os.path.join(cfgdir, 'servers')

    # create the directory
    if not os.path.isdir(cfgdir):
        os.makedirs(cfgdir)

    # define default config file contents if none provided
    if config_contents is None:
        config_contents = """
#
[auth]
password-stores =

[miscellany]
interactive-conflicts = false
"""

    # define default server file contents if none provided
    if server_contents is None:
        http_library_str = ""
        server_contents = """
#
[global]
%s
store-plaintext-passwords=yes
store-passwords=yes
""" % (http_library_str)

    file_write(cfgfile_cfg, config_contents)
    file_write(cfgfile_srv, server_contents)

def _with_config_dir(args):
    if '--config-dir' in args:
        return args
    else:
        return args + ('--config-dir', default_config_dir)

def _with_auth(args):
    assert '--password' not in args
    args = args + ('--password', wc_passwd,
                   '--no-auth-cache' )
    if '--username' in args:
        return args
    else:
        return args + ('--username', wc_author )

# For running subversion and returning the output
def run_hg(error_expected, *varargs):
    """Run hg with VARARGS; return exit code as int; stdout, stderr as
    lists of lines (including line terminators).
    If ERROR_EXPECTED is None, any stderr also will be printed.  If
    you're just checking that something does/doesn't come out of
    stdout/stderr, you might want to use actions.run_and_verify_svn()."""
    return run_command(hg_binary, error_expected, 0,
                       *(_with_auth(_with_config_dir(varargs))))

# def run_entriesdump(path):
#   """Run the entries-dump helper, returning a dict of Entry objects."""
#   # use spawn_process rather than run_command to avoid copying all the data
#   # to stdout in verbose mode.
#   exit_code, stdout_lines, stderr_lines = spawn_process(entriesdump_binary,
#                                                         0, 0, None, path)
#   if options.verbose:
#     ### finish the CMD output
#     print
#   if exit_code or stderr_lines:
#     ### report on this? or continue to just skip it?
#     return None
#
#   class Entry(object):
#     pass
#   entries = { }
#   exec(''.join([line for line in stdout_lines if not line.startswith("DBG:")]))
#   return entries


# Chmod recursively on a whole subtree
def chmod_tree(path, mode, mask):
    for dirpath, dirs, files in os.walk(path):
        for name in dirs + files:
            fullname = os.path.join(dirpath, name)
            if not os.path.islink(fullname):
                new_mode = (os.stat(fullname)[stat.ST_MODE] & ~mask) | mode
                os.chmod(fullname, new_mode)

# For clearing away working copies
def safe_rmtree(dirname, retry=0):
    "Remove the tree at DIRNAME, making it writable first"
    def rmtree(dirname):
        chmod_tree(dirname, 0o666, 0o666)
        shutil.rmtree(dirname)

    if not os.path.exists(dirname):
        return

    if retry:
        for delay in (0.5, 1, 2, 4):
            try:
                rmtree(dirname)
                break
            except:
                time.sleep(delay)
        else:
            rmtree(dirname)
    else:
        rmtree(dirname)

# For making local mods to files
def file_append(path, new_text):
    "Append NEW_TEXT to file at PATH"
    open(path, 'a').write(new_text)

# Append in binary mode
def file_append_binary(path, new_text):
    "Append NEW_TEXT to file at PATH in binary mode"
    open(path, 'ab').write(new_text)

# For creating new files, and making local mods to existing files.
def file_write(path, contents, mode='w'):
    """Write the CONTENTS to the file at PATH, opening file using MODE,
    which is (w)rite by default."""
    open(path, mode).write(contents)

# For replacing parts of contents in an existing file, with new content.
def file_substitute(path, contents, new_contents):
    """Replace the CONTENTS in the file at PATH using the NEW_CONTENTS"""
    fcontent = open(path, 'r').read().replace(contents, new_contents)
    open(path, 'w').write(fcontent)

# For creating blank new repositories
def create_repos(path):
    """Create a brand-new SVN repository at PATH.  If PATH does not yet
    exist, create it."""

    if not os.path.exists(path):
        os.makedirs(path) # this creates all the intermediate dirs, if neccessary

    opts = ("--bdb-txn-nosync",)
    if options.server_minor_version < 5:
        opts += ("--pre-1.5-compatible",)
    elif options.server_minor_version < 6:
        opts += ("--pre-1.6-compatible",)
    elif options.server_minor_version < 7:
        opts += ("--pre-1.7-compatible",)
    if options.fs_type is not None:
        opts += ("--fs-type=" + options.fs_type,)
    exit_code, stdout, stderr = run_command(svnadmin_binary, 1, 0, "create",
                                            path, *opts)

    # Skip tests if we can't create the repository.
    if stderr:
        for line in stderr:
            if line.find('Unknown FS type') != -1:
                raise Skip
        # If the FS type is known, assume the repos couldn't be created
        # (e.g. due to a missing 'svnadmin' binary).
        raise SVNRepositoryCreateFailure("".join(stderr).rstrip())

    # Allow unauthenticated users to write to the repos, for ra_svn testing.
    file_write(get_svnserve_conf_file_path(path),
               "[general]\nauth-access = write\n");
    if options.enable_sasl:
        file_append(get_svnserve_conf_file_path(path),
                    "realm = cvs2hg_test\n[sasl]\nuse-sasl = true\n")
    else:
        file_append(get_svnserve_conf_file_path(path), "password-db = passwd\n")
        file_append(os.path.join(path, "conf", "passwd"),
                    "[users]\njrandom = rayjandom\njconstant = rayjandom\n");

    if options.fs_type is None or options.fs_type == 'fsfs':
        # fsfs.conf file
        if options.config_file is not None:
            shutil.copy(options.config_file, get_fsfs_conf_file_path(path))

        # format file
        if options.fsfs_sharding is not None:
            def transform_line(line):
                if line.startswith('layout '):
                    if options.fsfs_sharding > 0:
                        line = 'layout sharded %d' % options.fsfs_sharding
                    else:
                        line = 'layout linear'
                return line

            # read it
            format_file_path = get_fsfs_format_file_path(path)
            contents = open(format_file_path, 'rb').read()

            # tweak it
            new_contents = "".join([transform_line(line) + "\n"
                                    for line in contents.split("\n")])
            if new_contents[-1] == "\n":
                # we don't currently allow empty lines (\n\n) in the format file.
                new_contents = new_contents[:-1]

            # replace it
            os.chmod(format_file_path, 0o666)
            file_write(format_file_path, new_contents, 'wb')

    # make the repos world-writeable, for mod_dav_svn's sake.
    chmod_tree(path, 0o666, 0o666)

# For copying a repository
def copy_repos(src_path, dst_path, head_revision, ignore_uuid = 1):
    "Copy the repository SRC_PATH, with head revision HEAD_REVISION, to DST_PATH"

    # Save any previous value of SVN_DBG_QUIET
    saved_quiet = os.environ.get('SVN_DBG_QUIET')
    os.environ['SVN_DBG_QUIET'] = 'y'

    # Do an svnadmin dump|svnadmin load cycle. Print a fake pipe command so that
    # the displayed CMDs can be run by hand
    create_repos(dst_path)
    dump_args = ['dump', src_path]
    load_args = ['load', dst_path]

    if ignore_uuid:
        load_args = load_args + ['--ignore-uuid']
    if options.verbose:
        sys.stdout.write('CMD: %s %s | %s %s\n' %
                         (os.path.basename(svnadmin_binary), ' '.join(dump_args),
                          os.path.basename(svnadmin_binary), ' '.join(load_args)))
        sys.stdout.flush()
    start = time.time()

    dump_in, dump_out, dump_err, dump_kid = open_pipe(
      [svnadmin_binary] + dump_args)
    load_in, load_out, load_err, load_kid = open_pipe(
      [svnadmin_binary] + load_args,
      stdin=dump_out) # Attached to dump_kid

    stop = time.time()
    if options.verbose:
        print('<TIME = %.6f>' % (stop - start))

    load_stdout, load_stderr, load_exit_code = wait_on_pipe(load_kid, True)
    dump_stdout, dump_stderr, dump_exit_code = wait_on_pipe(dump_kid, True)

    dump_in.close()
    dump_out.close()
    dump_err.close()
    #load_in is dump_out so it's already closed.
    load_out.close()
    load_err.close()

    if saved_quiet is None:
        del os.environ['SVN_DBG_QUIET']
    else:
        os.environ['SVN_DBG_QUIET'] = saved_quiet

    dump_re = re.compile(r'^\* Dumped revision (\d+)\.\r?$')
    expect_revision = 0
    for dump_line in dump_stderr:
        match = dump_re.match(dump_line)
        if not match or match.group(1) != str(expect_revision):
            print('ERROR:  dump failed: %s' % dump_line.strip())
            raise SVNRepositoryCopyFailure
        expect_revision += 1
    if expect_revision != head_revision + 1:
        print('ERROR:  dump failed; did not see revision %s' % head_revision)
        raise SVNRepositoryCopyFailure

    load_re = re.compile(r'^------- Committed revision (\d+) >>>\r?$')
    expect_revision = 1
    for load_line in load_stdout:
        match = load_re.match(load_line)
        if match:
            if match.group(1) != str(expect_revision):
                print('ERROR:  load failed: %s' % load_line.strip())
                raise SVNRepositoryCopyFailure
            expect_revision += 1
    if expect_revision != head_revision + 1:
        print('ERROR:  load failed; did not see revision %s' % head_revision)
        raise SVNRepositoryCopyFailure


def canonicalize_url(input):
    "Canonicalize the url, if the scheme is unknown, returns intact input"

    m = re.match(r"^((file://)|((svn|svn\+ssh|http|https)(://)))", input)
    if m:
        scheme = m.group(1)
        return scheme + re.sub(r'//*', '/', input[len(scheme):])
    else:
        return input


def create_python_hook_script(hook_path, hook_script_code):
    """Create a Python hook script at HOOK_PATH with the specified
       HOOK_SCRIPT_CODE."""

    if windows:
        # Use an absolute path since the working directory is not guaranteed
        hook_path = os.path.abspath(hook_path)
        # Fill the python file.
        file_write("%s.py" % hook_path, hook_script_code)
        # Fill the batch wrapper file.
        file_append("%s.bat" % hook_path,
                    "@\"%s\" %s.py %%*\n" % (sys.executable, hook_path))
    else:
        # For all other platforms
        file_write(hook_path, "#!%s\n%s" % (sys.executable, hook_script_code))
        os.chmod(hook_path, 0o755)

def write_restrictive_svnserve_conf(repo_dir, anon_access="none"):
    "Create a restrictive authz file ( no anynomous access )."

    fp = open(get_svnserve_conf_file_path(repo_dir), 'w')
    fp.write("[general]\nanon-access = %s\nauth-access = write\n"
             "authz-db = authz\n" % anon_access)
    if options.enable_sasl:
        fp.write("realm = cvs2hg_test\n[sasl]\nuse-sasl = true\n");
    else:
        fp.write("password-db = passwd\n")
    fp.close()

# Warning: because mod_dav_svn uses one shared authz file for all
# repositories, you *cannot* use write_authz_file in any test that
# might be run in parallel.
#
# write_authz_file can *only* be used in test suites which disable
# parallel execution at the bottom like so
#   if __name__ == '__main__':
#     cvs2hg_test.main.run_tests(test_list, serial_only = True)
def write_authz_file(sbox, rules, sections=None):
    """Write an authz file to SBOX, appropriate for the RA method used,
  with authorizations rules RULES mapping paths to strings containing
  the rules. You can add sections SECTIONS (ex. groups, aliases...) with
  an appropriate list of mappings.
  """
    fp = open(sbox.authz_file, 'w')

    # When the sandbox repository is read only it's name will be different from
    # the repository name.
    repo_name = sbox.repo_dir
    while repo_name[-1] == '/':
        repo_name = repo_name[:-1]
    repo_name = os.path.basename(repo_name)

    if sbox.repo_url.startswith("http"):
        prefix = repo_name + ":"
    else:
        prefix = ""
    if sections:
        for p, r in sections.items():
            fp.write("[%s]\n%s\n" % (p, r))

    for p, r in rules.items():
        fp.write("[%s%s]\n%s\n" % (prefix, p, r))
    fp.close()

def use_editor(func):
    os.environ['SVN_EDITOR'] = svneditor_script
    os.environ['SVN_MERGE'] = svneditor_script
    os.environ['SVNTEST_EDITOR_FUNC'] = func
    os.environ['SVN_TEST_PYTHON'] = sys.executable

def mergeinfo_notify_line(revstart, revend):
    """Return an expected output line that describes the beginning of a
    mergeinfo recording notification on revisions REVSTART through REVEND."""
    if (revend is None):
        if (revstart < 0):
            revstart = abs(revstart)
            return "--- Recording mergeinfo for reverse merge of r%ld .*:\n" \
                   % (revstart)
        else:
            return "--- Recording mergeinfo for merge of r%ld .*:\n" % (revstart)
    elif (revstart < revend):
        return "--- Recording mergeinfo for merge of r%ld through r%ld .*:\n" \
               % (revstart, revend)
    else:
        return "--- Recording mergeinfo for reverse merge of r%ld through " \
               "r%ld .*:\n" % (revstart, revend)

def merge_notify_line(revstart=None, revend=None, same_URL=True,
                      foreign=False):
    """Return an expected output line that describes the beginning of a
    merge operation on revisions REVSTART through REVEND.  Omit both
    REVSTART and REVEND for the case where the left and right sides of
    the merge are from different URLs."""
    from_foreign_phrase = foreign and "\(from foreign repository\) " or ""
    if not same_URL:
        return "--- Merging differences between %srepository URLs into '.+':\n" \
               % (foreign and "foreign " or "")
    if revend is None:
        if revstart is None:
            # The left and right sides of the merge are from different URLs.
            return "--- Merging differences between %srepository URLs into '.+':\n" \
                   % (foreign and "foreign " or "")
        elif revstart < 0:
            return "--- Reverse-merging %sr%ld into '.+':\n" \
                   % (from_foreign_phrase, abs(revstart))
        else:
            return "--- Merging %sr%ld into '.+':\n" \
                   % (from_foreign_phrase, revstart)
    else:
        if revstart > revend:
            return "--- Reverse-merging %sr%ld through r%ld into '.+':\n" \
                   % (from_foreign_phrase, revstart, revend)
        else:
            return "--- Merging %sr%ld through r%ld into '.+':\n" \
                   % (from_foreign_phrase, revstart, revend)


def make_log_msg():
    "Conjure up a log message based on the calling test."

    for idx in range(1, 100):
        frame = sys._getframe(idx)

        # If this frame isn't from a function in *_tests.py, then skip it.
        filename = frame.f_code.co_filename
        if not filename.endswith('_tests.py'):
            continue

        # There should be a test_list in this module.
        test_list = frame.f_globals.get('test_list')
        if test_list is None:
            continue

        # If the function is not in the test_list, then skip it.
        func_name = frame.f_code.co_name
        func_ob = frame.f_globals.get(func_name)
        if func_ob not in test_list:
            continue

        # Make the log message look like a line from a traceback.
        # Well...close. We use single quotes to avoid interfering with the
        # double-quote quoting performed on Windows
        return "File '%s', line %d, in %s" % (filename, frame.f_lineno, func_name)


######################################################################
# Functions which check the test configuration
# (useful for conditional XFails)

def is_os_windows():
    return os.name == 'nt'

def is_posix_os():
    return os.name == 'posix'

def is_os_darwin():
    return sys.platform == 'darwin'

def is_fs_case_insensitive():
    return (is_os_darwin() or is_os_windows())

def server_has_mergeinfo():
    return options.server_minor_version >= 5

def server_has_revprop_commit():
    return options.server_minor_version >= 5

def server_sends_copyfrom_on_update():
    return options.server_minor_version >= 5

def server_authz_has_aliases():
    return options.server_minor_version >= 5

def server_gets_client_capabilities():
    return options.server_minor_version >= 5

def server_has_partial_replay():
    return options.server_minor_version >= 5

def server_enforces_date_syntax():
    return options.server_minor_version >= 5

######################################################################


class TestSpawningThread(threading.Thread):
    """A thread that runs test cases in their own processes.
    Receives test numbers to run from the queue, and saves results into
    the results field."""
    def __init__(self, queue):
        threading.Thread.__init__(self)
        self.queue = queue
        self.results = []

    def run(self):
        while True:
            try:
                next_index = self.queue.get_nowait()
            except queue.Empty:
                return

            self.run_one(next_index)

    def run_one(self, index):
        command = sys.argv[0]

        args = []
        args.append(str(index))
        args.append('-c')
        # add some startup arguments from this process
        if options.verbose:
            args.append('-v')
        if options.cleanup:
            args.append('--cleanup')

        result, stdout_lines, stderr_lines = spawn_process(command, 0, 0, None,
                                                           *args)
        self.results.append((index, result, stdout_lines, stderr_lines))

        if result != 1:
            sys.stdout.write('.')
        else:
            sys.stdout.write('F')

        sys.stdout.flush()

class TestRunner:
    """Encapsulate a single test case (predicate), including logic for
    runing the test and test list output."""

    def __init__(self, func, index):
        self.pred = cvs2hg_test.testcase.create_test_case(func)
        self.index = index

    def list(self):
        if options.verbose and self.pred.inprogress:
            print(" %2d     %-5s  %s [[%s]]" % (self.index,
                                              self.pred.list_mode(),
                                              self.pred.description,
                                              self.pred.inprogress))
        else:
            print(" %2d     %-5s  %s" % (self.index,
                                         self.pred.list_mode(),
                                         self.pred.description))
        sys.stdout.flush()

    def get_function_name(self):
        return self.pred.get_function_name()

    def _print_name(self, prefix):
        if self.pred.inprogress:
            print("%s %s %s: %s [[WIMP: %s]]" % (prefix,
                                                 os.path.basename(sys.argv[0]),
                                                 str(self.index),
                                                 self.pred.description,
                                                 self.pred.inprogress))
        else:
            print("%s %s %s: %s" % (prefix,
                                    os.path.basename(sys.argv[0]),
                                    str(self.index),
                                    self.pred.description))
        sys.stdout.flush()

    def run(self):
        """Run self.pred and return the result.  The return value is
            - 0 if the test was successful
            - 1 if it errored in a way that indicates test failure
            - 2 if the test skipped
            """
        sbox_name = self.pred.get_sandbox_name()
        if sbox_name:
            sandbox = cvs2hg_test.sandbox.Sandbox(sbox_name, self.index)
        else:
            sandbox = None

        # Explicitly set this so that commands that commit but don't supply a
        # log message will fail rather than invoke an editor.
        # Tests that want to use an editor should invoke cvs2hg_test.main.use_editor.
        os.environ['SVN_EDITOR'] = ''
        os.environ['SVNTEST_EDITOR_FUNC'] = ''

        saved_dir = os.getcwd()
        try:
            rc = self.pred.run(sandbox)
            if rc is not None:
                self._print_name('STYLE ERROR in')
                print('Test driver returned a status code.')
                sys.exit(255)
            result = cvs2hg_test.testcase.RESULT_OK
        except Skip as ex:
            result = cvs2hg_test.testcase.RESULT_SKIP
        except Failure as ex:
            result = cvs2hg_test.testcase.RESULT_FAIL
            # We captured Failure and its subclasses. We don't want to print
            # anything for plain old Failure since that just indicates test
            # failure, rather than relevant information. However, if there
            # *is* information in the exception's arguments, then print it.
            if ex.__class__ != Failure or ex.args:
                ex_args = str(ex)
                if ex_args:
                    print('EXCEPTION: %s: %s' % (ex.__class__.__name__, ex_args))
                else:
                    print('EXCEPTION: %s' % ex.__class__.__name__)
            traceback.print_exc(file=sys.stdout)
            sys.stdout.flush()
        except KeyboardInterrupt:
            print('Interrupted')
            sys.exit(0)
        except SystemExit as ex:
            print('EXCEPTION: SystemExit(%d), skipping cleanup' % ex.code)
            self._print_name(ex.code and 'FAIL: ' or 'PASS: ')
            raise
        except:
            result = cvs2hg_test.testcase.RESULT_FAIL
            print('UNEXPECTED EXCEPTION:')
            traceback.print_exc(file=sys.stdout)
            sys.stdout.flush()

        os.chdir(saved_dir)
        exit_code, result_text, result_benignity = self.pred.results(result)
        if not (options.quiet and result_benignity):
            self._print_name(result_text)
        if sandbox is not None and exit_code != 1 and options.cleanup:
            sandbox.cleanup_test_paths()
        return exit_code

######################################################################
# Main testing functions

# These two functions each take a TEST_LIST as input.  The TEST_LIST
# should be a list of test functions; each test function should take
# no arguments and return a 0 on success, non-zero on failure.
# Ideally, each test should also have a short, one-line docstring (so
# it can be displayed by the 'list' command.)

# Func to run one test in the list.
def run_one_test(n, test_list, finished_tests = None):
    """Run the Nth client test in TEST_LIST, return the result.

    If we're running the tests in parallel spawn the test in a new process.
    """

    if (n < 1) or (n > len(test_list) - 1):
        print("There is no test %s.\n" % n)
        return 1

    # Run the test.
    exit_code = TestRunner(test_list[n], n).run()
    return exit_code

def _internal_run_tests(test_list, testnums, parallel, progress_func):
    """Run the tests from TEST_LIST whose indices are listed in TESTNUMS.

    If we're running the tests in parallel spawn as much parallel processes
    as requested and gather the results in a temp. buffer when a child
    process is finished.
    """

    exit_code = 0
    finished_tests = []
    tests_started = 0

    if not parallel:
        for i, testnum in enumerate(testnums):
            if run_one_test(testnum, test_list) == 1:
                exit_code = 1
            # signal progress
            if progress_func:
                progress_func(i+1, len(testnums))
    else:
        number_queue = queue.Queue()
        for num in testnums:
            number_queue.put(num)

        threads = [ TestSpawningThread(number_queue) for i in range(parallel) ]
        for t in threads:
            t.start()

        for t in threads:
            t.join()

        # list of (index, result, stdout, stderr)
        results = []
        for t in threads:
            results += t.results
        results.sort()

        # signal some kind of progress
        if progress_func:
            progress_func(len(testnums), len(testnums))

        # terminate the line of dots
        print("")

        # all tests are finished, find out the result and print the logs.
        for (index, result, stdout_lines, stderr_lines) in results:
            if stdout_lines:
                for line in stdout_lines:
                    sys.stdout.write(line)
            if stderr_lines:
                for line in stderr_lines:
                    sys.stdout.write(line)
            if result == 1:
                exit_code = 1

    cvs2hg_test.sandbox.cleanup_deferred_test_paths()
    return exit_code


def create_default_options():
    """Set the global options to the defaults, as provided by the argument
       parser."""
    _parse_options([])


def _create_parser():
    """Return a parser for our test suite."""
    # set up the parser
    usage = 'usage: %prog [options] [<test> ...]'
    parser = optparse.OptionParser(usage=usage)
    parser.add_option('-l', '--list', action='store_true', dest='list_tests',
                      help='Print test doc strings instead of running them')
    parser.add_option('-v', '--verbose', action='store_true',
                      help='Print binary command-lines (not with --quiet)')
    parser.add_option('-q', '--quiet', action='store_true',
                      help='Print only unexpected results (not with --verbose)')
    parser.add_option('-p', '--parallel', action='store_const', const=5,
                      dest='parallel',
                      help='Run the tests in parallel')
    parser.add_option('-c', action='store_true', dest='is_child_process',
                      help='Flag if we are running this python test as a ' +
                           'child process')
    parser.add_option('--cleanup', action='store_true',
                      help='Whether to clean up')
    parser.add_option('--bin', action='store', dest='hg_bin',
                      help='Use given hg binary')
    parser.add_option('--config-file', action='store',
                      help="Configuration file for tests.")
    parser.add_option('--keep-local-tmp', action='store_true',
                      help="Don't remove test-temporary-work-dir/local_tmp after test " +
                           "run is complete.  Useful for debugging failures.")
    parser.add_option('--development', action='store_true',
                      help='Test development mode: provides more detailed ' +
                           'test output and ignores all exceptions in the ' +
                           'run_and_verify* functions. This option is only ' +
                           'useful during test development!')

    # most of the defaults are None, but some are other values, set them here
    parser.set_defaults(
          # bin='/usr/bin/hg',
    )

    return parser


def _parse_options(arglist=sys.argv[1:]):
    """Parse the arguments in arg_list, and set the global options object with
       the results"""

    global options

    parser = _create_parser()
    (options, args) = parser.parse_args(arglist)

    # some sanity checking
    if options.verbose and options.quiet:
        parser.error("'verbose' and 'quiet' are incompatible")

    return (parser, args)


def run_tests(test_list, serial_only = False):
    """Main routine to run all tests in TEST_LIST.

    NOTE: this function does not return. It does a sys.exit() with the
          appropriate exit code.
    """

    sys.exit(execute_tests(test_list, serial_only))


# Main func.  This is the "entry point" that all the test scripts call
# to run their list of tests.
#
# This routine parses sys.argv to decide what to do.
def execute_tests(test_list, serial_only = False, test_name = None,
                  progress_func = None):
    """Similar to run_tests(), but just returns the exit code, rather than
    exiting the process.  This function can be used when a caller doesn't
    want the process to die."""

    # global pristine_url
    global hg_binary
    global options

    if test_name:
        sys.argv[0] = test_name

    testnums = []

    if not options:
        (parser, args) = _parse_options()
    else:
        args = []
        parser = _create_parser()

    # parse the positional arguments (test nums, names)
    for arg in args:
        appended = False
        try:
            testnums.append(int(arg))
            appended = True
        except ValueError:
            # Do nothing for now.
            appended = False

        if not appended:
            try:
                # Check if the argument is a range
                numberstrings = arg.split(':');
                if len(numberstrings) != 2:
                    numberstrings = arg.split('-');
                    if len(numberstrings) != 2:
                        raise ValueError
                left = int(numberstrings[0])
                right = int(numberstrings[1])
                if left > right:
                    raise ValueError

                for nr in range(left,right+1):
                    testnums.append(nr)
                else:
                    appended = True
            except ValueError:
                appended = False

        if not appended:
            try:
                # Check if the argument is a function name, and translate
                # it to a number if possible
                for testnum in list(range(1, len(test_list))):
                    test_case = TestRunner(test_list[testnum], testnum)
                    if test_case.get_function_name() == str(arg):
                        testnums.append(testnum)
                        appended = True
                        break
            except ValueError:
                appended = False

        if not appended:
            parser.error("invalid test number, range of numbers, " +
                         "or function '%s'\n" % arg)

    # Calculate pristine_url from test_area_url.
    # pristine_url = options.test_area_url + '/' + pathname2url(pristine_dir)

    if options.hg_bin:
        hg_binary = options.hg_bin

    ######################################################################

    # Cleanup: if a previous run crashed or interrupted the python
    # interpreter, then `temp_dir' was never removed.  This can cause wonkiness.
    if not options.is_child_process:
        safe_rmtree(temp_dir, 1)

    if not testnums:
        # If no test numbers were listed explicitly, include all of them:
        testnums = list(range(1, len(test_list)))

    if options.list_tests:
        print("Test #  Mode   Test Description")
        print("------  -----  ----------------")
        for testnum in testnums:
            TestRunner(test_list[testnum], testnum).list()

        # done. just exit with success.
        sys.exit(0)

    # don't run tests in parallel when the tests don't support it or there
    # are only a few tests to run.
    if serial_only or len(testnums) < 2:
        options.parallel = 0

    if not options.is_child_process:
        # Build out the default configuration directory
        create_config_dir(default_config_dir)

        # Setup the pristine repository
        # cvs2hg_test.actions.setup_pristine_repository()

    # Run the tests.
    exit_code = _internal_run_tests(test_list, testnums, options.parallel,
                                    progress_func)

    # Remove all scratchwork: the 'pristine' repository, greek tree, etc.
    # This ensures that an 'import' will happen the next time we run.
    if not options.is_child_process and not options.keep_local_tmp:
        safe_rmtree(temp_dir, 1)

    # Cleanup after ourselves.
    cvs2hg_test.sandbox.cleanup_deferred_test_paths()

    # Return the appropriate exit code from the tests.
    return exit_code
