# Copyright (c) 2020, Nordic Semiconductor ASA

import collections
import os
import re
import shutil
import subprocess
import sys
import textwrap
from pathlib import Path, PurePath

import pytest
from conftest import (
    GIT,
    WINDOWS,
    add_commit,
    add_tag,
    check_output,
    check_proj_consistency,
    cmd,
    cmd_raises,
    create_branch,
    create_repo,
    create_workspace,
    rev_parse,
)

from west.manifest import ImportFlag as MIF
from west.manifest import Manifest, ManifestImportFailed, ManifestProject, Project

assert 'TOXTEMPDIR' in os.environ, "you must run these tests using tox"

#
# Helpers
#

# A container for the remote locations of the repositories involved in
# a west update. These attributes may be None.
UpdateRemotes = collections.namedtuple('UpdateRemotes',
                                       'net_tools kconfiglib tagged_repo')

# A container type which holds the remote and local repository paths
# used in tests of 'west update'. This also contains manifest-rev
# branches and HEAD commits before (attributes ending in _0) and after
# (ending in _1) the 'west update' is done by update_helper().
#
# These may be None.
#
# See conftest.py for details on the manifest used in west update
# testing, and update_helper() below as well.
UpdateResults = collections.namedtuple('UpdateResults',
                                       'nt_remote nt_local '
                                       'kl_remote kl_local '
                                       'tr_remote tr_local '
                                       'nt_mr_0 nt_mr_1 '
                                       'kl_mr_0 kl_mr_1 '
                                       'tr_mr_0 tr_mr_1 '
                                       'nt_head_0 nt_head_1 '
                                       'kl_head_0 kl_head_1 '
                                       'tr_head_0 tr_head_1')

# Helper list for forming commands that add submodules from
# remotes which may be on the local file system. Such cases
# must be explicitly authorized since git 2.28.1. For details,
# see:
#
# https://github.blog/2022-10-18-git-security-vulnerabilities-announced/#cve-2022-39253
SUBMODULE_ADD = [GIT,
                 '-c', 'protocol.file.allow=always',
                 'submodule',
                 'add']

# Helper string for the same purpose when running west update.
PROTOCOL_FILE_ALLOW = '--submodule-init-config protocol.file.allow=always'

#
# Test fixtures
#

@pytest.fixture
def west_update_tmpdir(west_init_tmpdir):
    '''Like west_init_tmpdir, but also runs west update.'''
    cmd('update', cwd=str(west_init_tmpdir))
    return west_init_tmpdir

#
# Test cases
#

def _list_f(format):
    return ['list', '-f', format]


def _match_multiline_regex(expected, actual):
    for eline_re, aline in zip(expected, actual):
        assert re.match(eline_re, aline) is not None, (aline, eline_re)


def test_workspace(west_update_tmpdir):
    # Basic test that west_update_tmpdir bootstrapped correctly. This
    # is a basic test of west init and west update.

    # Make sure the expected files and directories exist in the right
    # places.
    wct = west_update_tmpdir
    assert wct.check(dir=1)
    assert wct.join('subdir', 'Kconfiglib').check(dir=1)
    assert wct.join('subdir', 'Kconfiglib', '.git').check(dir=1)
    assert wct.join('subdir', 'Kconfiglib', 'kconfiglib.py').check(file=1)
    assert wct.join('tagged_repo').check(dir=1)
    assert wct.join('tagged_repo', '.git').check(dir=1)
    assert wct.join('tagged_repo', 'test.txt').check()
    assert wct.join('net-tools').check(dir=1)
    assert wct.join('net-tools', '.git').check(dir=1)
    assert wct.join('net-tools', 'qemu-script.sh').check(file=1)
    assert wct.join('zephyr').check(dir=1)
    assert wct.join('zephyr', '.git').check(dir=1)
    assert wct.join('zephyr', 'CODEOWNERS').check(file=1)
    assert wct.join('zephyr', 'include', 'header.h').check(file=1)
    assert wct.join('zephyr', 'subsys', 'bluetooth', 'code.c').check(file=1)


def test_list(west_update_tmpdir):
    # Projects shall be listed in the order they appear in the manifest.
    # Check the behavior for some format arguments of interest as well.
    actual = cmd(_list_f('{name} {revision} {path} {cloned} {clone_depth}'))
    expected = ['manifest HEAD zephyr cloned None',
                'Kconfiglib zephyr subdir/Kconfiglib cloned None',
                'tagged_repo v1.0 tagged_repo cloned None',
                'net-tools master net-tools cloned 1']
    assert actual.splitlines() == expected

    # We should be able to find projects by absolute or relative path
    # when outside any project. Invalid projects should error out.
    klib_rel = os.path.join('subdir', 'Kconfiglib')
    klib_abs = str(west_update_tmpdir.join('subdir', 'Kconfiglib'))

    rel_outside = cmd(_list_f('{name}') + [klib_rel]).strip()
    assert rel_outside == 'Kconfiglib'

    abs_outside = cmd(_list_f('{name}') + [klib_abs]).strip()
    assert abs_outside == 'Kconfiglib'

    rel_inside = cmd('list -f {name} .', cwd=klib_abs).strip()
    assert rel_inside == 'Kconfiglib'

    abs_inside = cmd(_list_f('{name}') + [klib_abs], cwd=klib_abs).strip()
    assert abs_inside == 'Kconfiglib'

    with pytest.raises(subprocess.CalledProcessError):
        cmd('list NOT_A_PROJECT', cwd=klib_abs)

    with pytest.raises(subprocess.CalledProcessError):
        cmd('list NOT_A_PROJECT')


def test_list_manifest(west_update_tmpdir):
    # The manifest's "self: path:" should only be used to print
    # path-type format strings with --manifest-path-from-yaml.

    os.mkdir('manifest_moved')
    shutil.copy('zephyr/west.yml', 'manifest_moved/west.yml')
    cmd('config manifest.path manifest_moved')

    path = cmd('list -f {path} manifest').strip()
    abspath = cmd('list -f {abspath} manifest').strip()
    posixpath = cmd('list -f {posixpath} manifest').strip()
    assert path == 'manifest_moved'
    assert Path(abspath) == west_update_tmpdir / 'manifest_moved'
    assert posixpath == Path(west_update_tmpdir).as_posix() + '/manifest_moved'

    path = cmd('list --manifest-path-from-yaml '
               '-f {path} manifest').strip()
    abspath = cmd('list --manifest-path-from-yaml '
                  '-f {abspath} manifest').strip()
    posixpath = cmd('list --manifest-path-from-yaml '
                    '-f {posixpath} manifest').strip()
    assert path == 'zephyr'
    assert Path(abspath) == Path(str(west_update_tmpdir / 'zephyr'))
    assert posixpath == Path(west_update_tmpdir).as_posix() + '/zephyr'


def test_list_groups(west_init_tmpdir):
    with open('zephyr/west.yml', 'w') as f:
        f.write("""
        manifest:
          defaults:
            remote: r
          remotes:
            - name: r
              url-base: https://example.com
          projects:
          - name: foo
            groups:
            - foo-group-1
            - foo-group-2
          - name: bar
            path: path-for-bar
          - name: baz
            groups:
            - baz-group
          group-filter: [-foo-group-1,-foo-group-2,-baz-group]
        """)

    def check(command_string, expected):
        out_lines = cmd(command_string).splitlines()
        assert out_lines == expected

    check(_list_f('{name} .{groups}. {path}'),
          ['manifest .. zephyr',
           'bar .. path-for-bar'])

    check(_list_f('{name} .{groups}. {path}') + ['foo'],
          ['foo .foo-group-1,foo-group-2. foo'])

    check(_list_f('{name} .{groups}. {path}') + ['baz'],
          ['baz .baz-group. baz'])

    check(_list_f("{name} .{groups}. {path}") + 'foo bar baz'.split(),
          ['foo .foo-group-1,foo-group-2. foo',
           'bar .. path-for-bar',
           'baz .baz-group. baz'])

    check(_list_f('{name} .{groups}. {path}') + ['--all'],
          ['manifest .. zephyr',
           'foo .foo-group-1,foo-group-2. foo',
           'bar .. path-for-bar',
           'baz .baz-group. baz'])

    check(_list_f('{name} .{groups}. {path} {active}') + ['--inactive'],
          ['foo .foo-group-1,foo-group-2. foo inactive',
           'baz .baz-group. baz inactive'])

    check(_list_f("{name} .{groups}. {path} {active}") + ['--all'] + 'foo bar'.split(),
          ['foo .foo-group-1,foo-group-2. foo inactive',
           'bar .. path-for-bar active'])

    err_msg = cmd_raises('list -i foo bar', subprocess.CalledProcessError)
    assert '-i cannot be combined with an explicit project list' in err_msg

    cmd('config manifest.group-filter +foo-group-1')
    check(_list_f('{name} .{groups}. {path} {active}'),
          ['manifest .. zephyr active',
           'foo .foo-group-1,foo-group-2. foo active',
           'bar .. path-for-bar active'])


def test_list_sha(west_update_tmpdir):
    # Regression test for listing with {sha}. This should print N/A
    # for the first project, which is the ManifestProject.

    assert cmd('list -f {sha}').startswith("N/A")


def test_manifest_untracked(west_update_tmpdir):

    def check(expected, cwd=None):
        out_lines = cmd("manifest --untracked", cwd=cwd).splitlines()
        assert out_lines == expected

    # Ensure that all projects are active
    projs = cmd('list -f {name}').splitlines()
    assert projs == ['manifest', 'Kconfiglib', 'tagged_repo', 'net-tools']

    # Disable Kconfiglib
    cmd('config manifest.group-filter -- -Kconfiglib-group')

    # Ensure that Kconfiglib is inactive
    projs = cmd('list -f {name}').splitlines()
    assert projs == ['manifest', 'tagged_repo', 'net-tools']
    projs = cmd('list --all -f {name}').splitlines()
    assert projs == ['manifest', 'Kconfiglib', 'tagged_repo', 'net-tools']

    topdir = Path(west_update_tmpdir)

    # No untracked files yet
    check(list())

    (topdir / "dir").mkdir()
    # Untracked dir
    check(['dir'])

    (topdir / "unt").mkdir()
    (topdir / "unt" / "file.yml").touch()
    # Ensure we show the dir as untracked, not the file
    check(['dir', 'unt'])

    # Add a file to see it's displayed correctly as untracked
    (topdir / "file.txt").touch()
    check(['dir', 'file.txt', 'unt'])
    (topdir / 'subdir' / "z.py").touch()
    check(['dir', 'file.txt', str(Path('subdir/z.py')), 'unt'])

    (topdir / "subdir" / "new").mkdir()
    (topdir / "subdir" / "new" / "afile.txt").touch()
    check(['dir', 'file.txt', str(Path('subdir/new')),
           str(Path('subdir/z.py')), 'unt'])

    # Check relative paths
    check([str(Path('../dir')), str(Path('../file.txt')),
           str(Path('../subdir/new')), str(Path('../subdir/z.py')), '.'],
          cwd=str(Path("unt/")))

    # Add a file to an existing project, ignored by --untracked
    (topdir / "net-tools" / "test_manifest_untracked.file").touch()
    check(['dir', 'file.txt', str(Path('subdir/new')), str(Path('subdir/z.py')),
           'unt'])

    kconfiglib = Path(topdir / "subdir" / "Kconfiglib")
    # Same but with an inactive project
    (kconfiglib / "test_manifest_untracked.file").touch()
    check(['dir', 'file.txt', str(Path('subdir/new')), str(Path('subdir/z.py')),
           'unt'])

    # Copy (via clone) a full Git repo so we verify that those are also
    # displayed as untracked.
    clone(topdir / "net-tools", Path('subdir/acopy'))
    clone(topdir / "net-tools", Path('tmpcopy'))

    check(['dir', 'file.txt', str(Path('subdir/acopy')), str(Path('subdir/new')),
           str(Path('subdir/z.py')), 'tmpcopy', 'unt'])

    # Empty a project so it's not a Git repo anymore
    (topdir / "net-tools" / ".git").rename(topdir / "net-tools" / "former-git")
    # Should make no difference
    check(['dir', 'file.txt', str(Path('subdir/acopy')), str(Path('subdir/new')),
           str(Path('subdir/z.py')), 'tmpcopy', 'unt'])

    # Same with an inactive project
    (kconfiglib / ".git").rename(kconfiglib / "former-git")
    # Should make no difference
    check(['dir', 'file.txt', str(Path('subdir/acopy')), str(Path('subdir/new')),
           str(Path('subdir/z.py')), 'tmpcopy', 'unt'])

    # Even if we make the whole inactive project disappear it should make no
    # difference at all, except that the renamed dir will show up.
    (kconfiglib).rename(topdir / "subdir" / "other")
    check(['dir', 'file.txt', str(Path('subdir/acopy')), str(Path('subdir/new')),
           str(Path('subdir/other')), str(Path('subdir/z.py')), 'tmpcopy', 'unt'])

@pytest.mark.skipif(sys.platform.startswith("win"), reason="symbolic links not tested on Windows")
def test_manifest_untracked_with_symlinks(west_update_tmpdir):

    def check(expected, cwd=None):
        out_lines = cmd("manifest --untracked", cwd=cwd).splitlines()
        assert out_lines == expected

    # Disable Kconfiglib to have an inactive project
    cmd('config manifest.group-filter -- -Kconfiglib-group')

    # Ensure that Kconfiglib is inactive
    projs = cmd('list -f {name}').splitlines()
    assert projs == ['manifest', 'tagged_repo', 'net-tools']

    # Create a folder symlink
    Path('asl').symlink_to(Path('subdir/Kconfiglib'))
    # Create another one
    Path('anothersl').symlink_to(Path('../'))
    # Yet another one
    Path('subdir/yetanothersl').symlink_to(Path('tagged_repo'))
    # check that symlinks are displayed like any other regular untracked file or
    # directory
    check(['anothersl', 'asl', str(Path('subdir/yetanothersl'))])

    # File symlink tests, should all be displayed as untracked as well
    Path('filesl.yml').symlink_to(Path('zephyr/west.yml'))
    Path('subdir/afsl.py').symlink_to(Path('net-tools/scripts/test.py'))
    check(['anothersl', 'asl', 'filesl.yml', str(Path('subdir/afsl.py')),
           str(Path('subdir/yetanothersl'))])

def test_manifest_freeze(west_update_tmpdir):
    # We should be able to freeze manifests.
    actual = cmd('manifest --freeze').splitlines()
    # Match the actual output against the expected line by line,
    # so failing lines can be printed in the test output.
    #
    # Since the actual remote URLs and SHAs are not predictable, we
    # don't match those precisely. However, we do expect:
    #
    # - the output to match project order as specified in our
    #   manifest
    # - attributes are listed in NURPCW order (name, url, ...)
    # - all revisions are full 40-character SHAs
    # - there isn't any random YAML tag crap
    expected_res = ['^manifest:$',
                    '^  projects:$',
                    '^  - name: Kconfiglib$',
                    '^    description: |',
                    '^      Kconfiglib is an implementation of$',
                    '^      the Kconfig language written in Python.$',
                    '^    url: .*$',
                    '^    revision: [a-f0-9]{40}$',
                    '^    path: subdir/Kconfiglib$',
                    '^    groups:$',
                    '^    - Kconfiglib-group$',
                    '^    submodules: true$',
                    '^  - name: tagged_repo$',
                    '^    url: .*$',
                    '^    revision: [a-f0-9]{40}$',
                    '^  - name: net-tools$',
                    '^    description: Networking tools.$',
                    '^    url: .*$',
                    '^    revision: [a-f0-9]{40}$',
                    '^    clone-depth: 1$',
                    '^    west-commands: scripts/west-commands.yml$',
                    '^  self:$',
                    '^    path: zephyr$']
    _match_multiline_regex(expected_res, actual)

def test_manifest_freeze_active(west_update_tmpdir):
    # We should be able to freeze manifests with inactive projects.
    cmd('config manifest.group-filter -- -Kconfiglib-group')

    actual = cmd('manifest --freeze --active-only').splitlines()
    # Same as test_manifest_freeze but without inactive projects
    expected_res = ['^manifest:$',
                    '^  projects:$',
                    '^  - name: tagged_repo$',
                    '^    url: .*$',
                    '^    revision: [a-f0-9]{40}$',
                    '^  - name: net-tools$',
                    '^    description: Networking tools.$',
                    '^    url: .*$',
                    '^    revision: [a-f0-9]{40}$',
                    '^    clone-depth: 1$',
                    '^    west-commands: scripts/west-commands.yml$',
                    '^  self:$',
                    '^    path: zephyr$']
    _match_multiline_regex(expected_res, actual)

def test_manifest_resolve(west_update_tmpdir):
    # We should be able to resolve manifests.
    actual = cmd('manifest --resolve').splitlines()
    # Similar as test_manifest_freeze but with resolved projects
    expected_res = ['^manifest:$',
                    '^  projects:$',
                    '^  - name: Kconfiglib$',
                    '^    description: |',
                    '^      Kconfiglib is an implementation of$',
                    '^      the Kconfig language written in Python.$',
                    '^    url: .*$',
                    '^    revision: zephyr$',
                    '^    path: subdir/Kconfiglib$',
                    '^    groups:$',
                    '^    - Kconfiglib-group$',
                    '^    submodules: true$',
                    '^  - name: tagged_repo$',
                    '^    url: .*$',
                    '^    revision: v1.0$',
                    '^  - name: net-tools$',
                    '^    description: Networking tools.$',
                    '^    url: .*$',
                    '^    revision: master$',
                    '^    clone-depth: 1$',
                    '^    west-commands: scripts/west-commands.yml$',
                    '^  self:$',
                    '^    path: zephyr$']
    _match_multiline_regex(expected_res, actual)

def test_manifest_resolve_active(west_update_tmpdir):
    # We should be able to resolve manifests with inactive projects.
    cmd('config manifest.group-filter -- -Kconfiglib-group')

    actual = cmd('manifest --resolve --active-only').splitlines()
    # Same as test_manifest_resolve but without inactive projects
    expected_res = ['^manifest:$',
                    '^  projects:$',
                    '^  - name: tagged_repo$',
                    '^    url: .*$',
                    '^    revision: v1.0$',
                    '^  - name: net-tools$',
                    '^    description: Networking tools.$',
                    '^    url: .*$',
                    '^    revision: master$',
                    '^    clone-depth: 1$',
                    '^    west-commands: scripts/west-commands.yml$',
                    '^  self:$',
                    '^    path: zephyr$']
    _match_multiline_regex(expected_res, actual)

def test_compare(config_tmpdir, west_init_tmpdir):
    # 'west compare' with no projects cloned should still work,
    # and not print anything.
    assert cmd('compare') == ''

    # Create an empty file and make sure the manifest repository
    # is included in 'west compare' output, and that we get some
    # information about the dirty tree from git.
    foo = west_init_tmpdir / 'zephyr' / 'foo'
    with open(foo, 'w'):
        pass
    actual = cmd('compare')
    assert actual.startswith('=== manifest')
    assert 'foo' in actual

    # --exit-code should work for the manifest repository too.
    with pytest.raises(subprocess.CalledProcessError):
        cmd('compare --exit-code')

    # Remove the file and verify compare output is empty again.
    os.unlink(foo)
    assert cmd('compare') == ''

    # Make sure project related output seems reasonable.
    cmd('update')
    kconfiglib = west_init_tmpdir / 'subdir' / 'Kconfiglib'
    bar = kconfiglib / 'bar'
    with open(bar, 'w'):
        pass
    actual = cmd('compare')
    assert actual.startswith('=== Kconfiglib (subdir/Kconfiglib):')
    assert 'bar' in actual

    # We shouldn't get any output for inactive projects by default, so
    # temporarily deactivate the Kconfiglib project and make sure that
    # works.
    cmd('config manifest.group-filter -- -Kconfiglib-group')
    assert cmd('compare') == ''
    # unless we ask for it with --all, or the project by name
    assert cmd('compare Kconfiglib').startswith(
        '=== Kconfiglib (subdir/Kconfiglib)')
    assert cmd('compare --all').startswith(
        '=== Kconfiglib (subdir/Kconfiglib)')
    # Activate the project again.
    cmd('config -d manifest.group-filter')

    # Verify --exit-code works as advertised, and clean up again.
    with pytest.raises(subprocess.CalledProcessError):
        cmd('compare --exit-code')
    os.unlink(bar)
    assert cmd('compare --exit-code') == ''

    # By default, a checked-out branch should print output, even if
    # the tree is otherwise clean...
    check_output(['git', 'checkout', '-b', 'mybranch'], cwd=kconfiglib)
    actual = cmd('compare')
    assert actual.startswith('=== Kconfiglib (subdir/Kconfiglib):')
    assert 'mybranch' in actual
    # unless we disable that explicitly...
    assert cmd('compare --ignore-branches') == ''
    # or the compare.ignore-branches configuration option is true...
    cmd('config compare.ignore-branches true')
    assert cmd('compare') == ''
    # unless we override that option on the command line.
    assert 'mybranch' in cmd('compare --no-ignore-branches')

def test_diff(west_init_tmpdir):
    # FIXME: Check output

    # Diff with no projects cloned shouldn't fail

    cmd('diff')
    cmd('diff --manifest')
    cmd('diff --stat')

    # Neither should it fail after fetching one or both projects

    cmd('update net-tools')
    cmd('diff')
    cmd('diff --stat')
    cmd('diff --manifest')

    cmd('update Kconfiglib')


def test_status(west_init_tmpdir):
    # FIXME: Check output

    # Status with no projects cloned shouldn't fail

    cmd('status')
    cmd('status --short --branch')

    # Neither should it fail after fetching one or both projects

    cmd('update net-tools')
    cmd('status')
    cmd('status --short --branch')

    cmd('update Kconfiglib')


def test_forall(west_init_tmpdir):
    # Note that the 'echo' command is available in both Unix shells
    # and Windows .bat files.

    # 'forall' with no projects cloned shouldn't fail

    assert cmd(['forall', '-c', 'echo foo']).splitlines() == [
        '=== running "echo foo" in manifest (zephyr):',
        'foo']

    # Neither should it fail after cloning one or both projects

    cmd('update net-tools')
    assert cmd(['forall', '-c', 'echo foo']).splitlines() == [
        '=== running "echo foo" in manifest (zephyr):',
        'foo',
        '=== running "echo foo" in net-tools (net-tools):',
        'foo']

    # Use environment variables

    env_var = "%WEST_PROJECT_NAME%" if WINDOWS else "$WEST_PROJECT_NAME"

    assert cmd(['forall', '-c', f'echo {env_var}']).splitlines() == [
        f'=== running "echo {env_var}" in manifest (zephyr):',
        'manifest',
        f'=== running "echo {env_var}" in net-tools (net-tools):',
        'net-tools']

    cmd('update Kconfiglib')
    assert cmd(['forall', '-c', 'echo foo']).splitlines() == [
        '=== running "echo foo" in manifest (zephyr):',
        'foo',
        '=== running "echo foo" in Kconfiglib (subdir/Kconfiglib):',
        'foo',
        '=== running "echo foo" in net-tools (net-tools):',
        'foo']

    assert cmd('forall --group Kconfiglib-group -c'.split() + ['echo foo']
               ).splitlines() == [
                   '=== running "echo foo" in Kconfiglib (subdir/Kconfiglib):',
                   'foo',
               ]


def test_grep(west_init_tmpdir):
    # Make sure we don't find things we don't expect, and do find
    # things we do.

    actual_before_update = cmd('grep net-').strip()
    actual_before_update_lines = actual_before_update.splitlines()
    assert len(actual_before_update_lines) == 2
    assert re.fullmatch(r'=== manifest \(zephyr\):',
                        actual_before_update_lines[0])
    assert re.search('net-tools',
                     actual_before_update_lines[1])

    assert not re.search('hello', cmd('grep hello'))

    cmd('update')
    assert re.search('hello', cmd('grep hello'))

    # Make sure '--' is handled properly: the first one is for
    # west, and the second one is for the tool

    assert re.search('west-commands', cmd('grep -- -- -commands'))


def test_update_projects(west_init_tmpdir):
    # Test the 'west update' command. It calls through to the same backend
    # functions that are used for automatic updates and 'west init'
    # reinitialization.

    # create local repositories
    cmd('update')

    # Add commits to the local repos.
    ur = update_helper(west_init_tmpdir)

    # We updated all the repositories, so all paths and commits should
    # be valid refs (i.e. there shouldn't be a None or empty string
    # value in a ur attribute).
    assert all(ur)

    # Make sure we see different manifest-rev commits and HEAD revisions,
    # except for the repository whose version was locked at a tag.
    assert ur.nt_mr_0 != ur.nt_mr_1, 'failed updating net-tools manifest-rev'
    assert ur.kl_mr_0 != ur.kl_mr_1, 'failed updating kconfiglib manifest-rev'
    assert ur.tr_mr_0 == ur.tr_mr_1, 'tagged_repo manifest-rev changed'
    assert ur.nt_head_0 != ur.nt_head_1, 'failed updating net-tools HEAD'
    assert ur.kl_head_0 != ur.kl_head_1, 'failed updating kconfiglib HEAD'
    assert ur.tr_head_0 == ur.tr_head_1, 'tagged_repo HEAD changed'

def test_update_projects_local_branch_commits(west_init_tmpdir):
    # Test the 'west update' command when working on local branch with local
    # commits and then updating project to upstream commit.
    # It calls through to the same backend functions that are used for
    # automatic updates and 'west init' reinitialization.

    # update all repositories
    cmd('update')

    # Create a local branch and add commits
    checkout_branch('net-tools', 'local_net_tools_test_branch', create=True)
    checkout_branch('subdir/Kconfiglib', 'local_kconfig_test_branch',
                    create=True)
    checkout_branch('tagged_repo', 'local_tagged_repo_test_branch',
                    create=True)
    add_commit('net-tools', 'test local branch commit', reconfigure=True)
    add_commit('subdir/Kconfiglib', 'test local branch commit',
               reconfigure=True)
    add_commit('tagged_repo', 'test local branch commit',
               reconfigure=True)
    net_tools_prev = head_subject('net-tools')
    kconfiglib_prev = head_subject('subdir/Kconfiglib')
    tagged_repo_prev = head_subject('tagged_repo')

    # Update the upstream repositories, getting an UpdateResults tuple
    # back.
    ur = update_helper(west_init_tmpdir)

    # We updated all the repositories, so all paths and commits should
    # be valid refs (i.e. there shouldn't be a None or empty string
    # value in a ur attribute).
    assert all(ur)

    # Verify each repository has moved to a new manifest-rev,
    # except tagged_repo, which has a manifest-rev locked to a tag.
    # Its HEAD should change, though.
    assert ur.nt_mr_0 != ur.nt_mr_1, 'failed updating net-tools manifest-rev'
    assert ur.kl_mr_0 != ur.kl_mr_1, 'failed updating kconfiglib manifest-rev'
    assert ur.tr_mr_0 == ur.tr_mr_1, 'tagged_repo manifest-rev changed'
    assert ur.nt_head_0 != ur.nt_head_1, 'failed updating net-tools HEAD'
    assert ur.kl_head_0 != ur.kl_head_1, 'failed updating kconfiglib HEAD'
    assert ur.tr_head_0 != ur.tr_head_1, 'failed updating tagged_repo HEAD'

    # Verify local branch is still present and untouched
    assert net_tools_prev != head_subject('net-tools')
    assert kconfiglib_prev != head_subject('subdir/Kconfiglib')
    assert tagged_repo_prev != head_subject('tagged_repo')
    checkout_branch('net-tools', 'local_net_tools_test_branch')
    checkout_branch('subdir/Kconfiglib', 'local_kconfig_test_branch')
    checkout_branch('tagged_repo', 'local_tagged_repo_test_branch')
    assert net_tools_prev == head_subject('net-tools')
    assert kconfiglib_prev == head_subject('subdir/Kconfiglib')
    assert tagged_repo_prev == head_subject('tagged_repo')

def test_update_tag_to_tag(west_init_tmpdir):
    # Verify we can update the tagged_repo repo to a new tag.

    # We only need to clone tagged_repo locally.
    cmd('update tagged_repo')

    def updater(remotes):
        # Create a v2.0 tag on the remote tagged_repo repository.
        add_commit(remotes.tagged_repo, 'another tagged_repo tagged commit')
        add_tag(remotes.tagged_repo, 'v2.0')

        # Update the manifest file to point the project's revision at
        # the new tag.
        manifest = Manifest.from_topdir(topdir=west_init_tmpdir)
        for p in manifest.projects:
            if p.name == 'tagged_repo':
                p.revision = 'v2.0'
                break
        else:
            raise AssertionError('no tagged_repo')
        with open(west_init_tmpdir / 'zephyr' / 'west.yml', 'w') as f:
            f.write(manifest.as_yaml())  # NOT as_frozen_yaml().

        # Pull down the v2.0 tag into west_init_tmpdir.
        cmd('update tagged_repo')
    ur = update_helper(west_init_tmpdir, updater=updater)

    # Make sure we have manifest-rev and HEADs before and after.
    assert ur.tr_mr_0
    assert ur.tr_mr_1
    assert ur.tr_head_0
    assert ur.tr_head_1

    # Make sure we have v1.0 and v2.0 tags locally.
    v1_0 = check_output([GIT, 'rev-parse', 'refs/tags/v1.0^{commit}'],
                        cwd=ur.tr_local)
    v2_0 = check_output([GIT, 'rev-parse', 'refs/tags/v2.0^{commit}'],
                        cwd=ur.tr_local)
    assert v1_0
    assert v2_0

    # Check that all the updates (including the first one in this
    # function) behaved correctly.
    assert ur.tr_mr_0 == v1_0
    assert ur.tr_mr_1 == v2_0
    assert ur.tr_head_0 == v1_0
    assert ur.tr_head_1 == v2_0

def test_update_head_0(west_init_tmpdir):
    # Verify that using HEAD~0 as the revision causes west to not touch the
    # local repo. In this case, local zephyr should remain unchanged, even
    # though it is referenced in the new top-level manifest 'my_manifest',
    # since it's referenced with HEAD~0. Check that a local commit remains, and
    # that local changes are kept as is. An invalid url ('na') is used to check
    # that west doesn't attempt to access it.
    # Expect only Kconfiglib to be updated, since it's mentioned in the new
    # manifest's name-allowlist.
    # Note: HEAD~0 is used instead of HEAD because HEAD causes west to instead
    # fetch the remote HEAD.
    # In git, HEAD is a reference, whereas HEAD~<n> is a valid revision but
    # not a reference. West fetches references, such as refs/heads/main or
    # HEAD, and commits not available locally, but will not fetch commits if
    # they are already available.
    # HEAD~0 is resolved to a specific commit that is locally available, and
    # therefore west will simply checkout the locally available commit,
    # identified by HEAD~0.

    # create local repositories
    cmd('update')

    local_zephyr = west_init_tmpdir / "zephyr"

    add_commit(local_zephyr, "local commit")
    local_commit = check_output([GIT, 'rev-parse', 'HEAD'], cwd=local_zephyr)

    with open(local_zephyr / "CODEOWNERS", 'a') as f:
        f.write("\n") # Make a local change (add a newline)

    my_manifest_dir = west_init_tmpdir / "my_manifest"
    my_manifest_dir.mkdir()

    my_manifest = Path(my_manifest_dir / "west.yml")
    my_manifest.write_text(
        '''
          manifest:
            projects:
              - name: zephyr
                revision: HEAD~0
                url: na
                import:
                  name-allowlist:
                    - Kconfiglib
        ''')

    cmd(["config", "manifest.path", my_manifest_dir])

    # Update the upstream repositories, getting an UpdateResults tuple
    # back.
    ur = update_helper(west_init_tmpdir)

    assert all(ur)

    assert ur.nt_mr_0 == ur.nt_mr_1, 'net-tools manifest-rev changed'
    assert ur.kl_mr_0 != ur.kl_mr_1, 'failed updating kconfiglib manifest-rev'
    assert ur.tr_mr_0 == ur.tr_mr_1, 'tagged_repo manifest-rev changed'
    assert ur.nt_head_0 == ur.nt_head_1, 'net-tools HEAD changed'
    assert ur.kl_head_0 != ur.kl_head_1, 'failed updating kconfiglib HEAD'
    assert ur.tr_head_0 == ur.tr_head_1, 'tagged_repo HEAD changed'

    local_commit2 = check_output([GIT, 'rev-parse', 'HEAD'], cwd=local_zephyr)
    modified_files = check_output([GIT, 'status', '--porcelain'],
                                  cwd=local_zephyr)

    assert local_commit == local_commit2, 'zephyr local commit changed'
    assert modified_files.strip() == "M CODEOWNERS", \
           'local zephyr change not preserved'

def test_update_some_with_imports(repos_tmpdir):
    # 'west update project1 project2' should work fine even when
    # imports are used, as long as the relevant projects are all
    # defined in the manifest repository.
    #
    # It currently should fail with a helpful message if the projects
    # are resolved via project imports.

    remotes = repos_tmpdir / 'repos'
    zephyr = remotes / 'zephyr'
    net_tools = remotes / 'net-tools'

    ws = repos_tmpdir / 'ws'
    create_workspace(ws)
    manifest_repo = ws / 'mp'
    create_repo(manifest_repo)
    add_commit(manifest_repo, 'manifest repo commit',
               # zephyr revision is implicitly master:
               files={'west.yml':
                      f'''
                      manifest:
                        projects:
                        - name: zephyr
                          url: {zephyr}
                          import: true
                        self:
                          import: foo.yml
                      ''',
                      'foo.yml':
                      f'''
                      manifest:
                        projects:
                        - name: net-tools
                          url: {net_tools}
                      '''})

    cmd(['init', '-l', manifest_repo])

    # Updating unknown projects should fail as always.

    with pytest.raises(subprocess.CalledProcessError):
        cmd('update unknown-project', cwd=ws)

    # Updating a list of projects when some are resolved via project
    # imports must fail.

    with pytest.raises(subprocess.CalledProcessError):
        cmd('update Kconfiglib net-tools', cwd=ws)

    # Updates of projects defined in the manifest repository or all
    # projects must succeed, and behave the same as if no imports
    # existed.

    cmd('update net-tools', cwd=ws)
    with pytest.raises(ManifestImportFailed):
        Manifest.from_topdir(topdir=ws)
    manifest = Manifest.from_topdir(topdir=ws,
                                    import_flags=MIF.IGNORE_PROJECTS)
    projects = manifest.get_projects(['net-tools', 'zephyr'])
    net_tools_project = projects[0]
    zephyr_project = projects[1]
    assert net_tools_project.is_cloned()
    assert not zephyr_project.is_cloned()

    cmd('update zephyr', cwd=ws)
    assert zephyr_project.is_cloned()

    cmd('update', cwd=ws)
    manifest = Manifest.from_topdir(topdir=ws)
    assert manifest.get_projects(['Kconfiglib'])[0].is_cloned()

def test_update_submodules_list(repos_tmpdir):
    # The west update command should not only update projects,
    # but also its submodules. Test uses two pairs of project
    # and submodule checking out submodule in the default
    # and custom location to verify both cases.

    # Repositories paths
    remotes = repos_tmpdir / 'repos'
    # tagged_repo is zephyr's submodule and will be updated
    # in the default location (i.e. zephyr/tagged_repo).
    zephyr = remotes / 'zephyr'
    tagged_repo = remotes / 'tagged_repo'
    # Kconfiglib is net_tools's submodule and will be updated
    # in the custom location (i.e. net-tools/third_parties/Kconfiglib).
    net_tools = remotes / 'net-tools'
    kconfiglib = remotes / 'Kconfiglib'
    kconfiglib_submodule = 'third_parties/Kconfiglib'

    # Creating west workspace and manifest repository.
    ws = repos_tmpdir / 'ws'
    create_workspace(ws)
    manifest_repo = ws / 'mp'
    create_repo(manifest_repo)

    # Commit west.yml describing projects and submodules dependencies.
    add_commit(manifest_repo, 'manifest repo commit',
               files={'west.yml':
                      f'''
                      manifest:
                        projects:
                        - name: zephyr
                          url: {zephyr}
                          submodules:
                            - path: tagged_repo
                        - name: net-tools
                          url: {net_tools}
                          submodules:
                            - name: Kconfiglib
                              path: {kconfiglib_submodule}
                       '''})
    cmd(['init', '-l', manifest_repo])

    # Make tagged_repo to be zephyr project submodule.
    subprocess.check_call(SUBMODULE_ADD +
                          [str(tagged_repo), 'tagged_repo'],
                          cwd=zephyr)
    # Commit changes to the zephyr repo.
    add_commit(zephyr, 'zephyr submodule change commit')

    # Make Kconfiglib to be net-tools project submodule.
    subprocess.check_call(SUBMODULE_ADD +
                          [str(kconfiglib), kconfiglib_submodule],
                          cwd=net_tools)
    # Commit changes to the net-tools repo.
    add_commit(net_tools, 'net-tools submodule change commit')

    # Get parsed data from the manifest.
    manifest = Manifest.from_topdir(topdir=ws,
                                    import_flags=MIF.IGNORE_PROJECTS)
    projects = manifest.get_projects(['zephyr', 'net-tools'])
    zephyr_project = projects[0]
    net_tools_project = projects[1]

    # Verify if tagged_repo submodule data are correct.
    assert zephyr_project.submodules
    assert zephyr_project.submodules[0].name is None
    assert zephyr_project.submodules[0].path == 'tagged_repo'
    # Verify if Kconfiglib submodule data are correct.
    assert net_tools_project.submodules
    assert net_tools_project.submodules[0].name == 'Kconfiglib'
    assert net_tools_project.submodules[0].path == kconfiglib_submodule

    # Check if projects repos are cloned - should not be.
    assert not zephyr_project.is_cloned()
    assert not net_tools_project.is_cloned()

    # Update only zephyr project.
    cmd(f'update {PROTOCOL_FILE_ALLOW} zephyr', cwd=ws)

    # Verify if only zephyr project was cloned.
    assert zephyr_project.is_cloned()
    assert not net_tools_project.is_cloned()

    # Verify if tagged-repo submodule was also cloned
    res = zephyr_project.git('rev-parse --show-cdup', check=False,
                             cwd=os.path.join(ws, 'zephyr', 'tagged_repo'),
                             capture_stderr=True, capture_stdout=True)
    assert not (res.returncode or res.stdout.strip())

    # Update all projects
    cmd(f'update {PROTOCOL_FILE_ALLOW}', cwd=ws)

    # Verify if both projects were cloned
    assert zephyr_project.is_cloned()
    assert net_tools_project.is_cloned()

    # Verify if Kconfiglib submodule was also cloned
    res = net_tools_project.git('rev-parse --show-cdup', check=False,
                                cwd=os.path.join(ws, 'net-tools',
                                                 kconfiglib_submodule),
                                capture_stderr=True, capture_stdout=True)
    assert not (res.returncode or res.stdout.strip())

    # Test freeze output with submodules
    # see test_manifest_freeze for details
    actual = cmd('manifest --freeze', cwd=ws).splitlines()
    expected_res = ['^manifest:$',
                    '^  projects:$',
                    '^  - name: zephyr$',
                    f'^    url: {re.escape(str(zephyr))}$',
                    '^    revision: [a-f0-9]{40}$',
                    '^    submodules:$',
                    '^    - path: tagged_repo$',
                    '^  - name: net-tools$',
                    f'^    url: {re.escape(str(net_tools))}$',
                    '^    revision: [a-f0-9]{40}$',
                    '^    submodules:$',
                    f'^    - path: {re.escape(str(kconfiglib_submodule))}$',
                    '^      name: Kconfiglib$',
                    '^  self:$',
                    '^    path: mp$']
    _match_multiline_regex(expected_res, actual)

def test_update_all_submodules(repos_tmpdir):
    # The west update command should not only update projects,
    # but also its submodules. Test verifies whether setting submodules
    # value to boolean True results in updating all project submodules
    # (even without specifying them implicitly as a list). Moreover it checks
    # if submodules are updated recursively.

    # Repositories paths
    remotes = repos_tmpdir / 'repos'
    # tagged_repo and net_tools are zephyr's submodules
    zephyr = remotes / 'zephyr'
    tagged_repo = remotes / 'tagged_repo'
    net_tools = remotes / 'net-tools'
    # Kconfiglib is net-tools submodule
    kconfiglib = remotes / 'Kconfiglib'

    # Creating west workspace and manifest repository.
    ws = repos_tmpdir / 'ws'
    create_workspace(ws)
    manifest_repo = ws / 'mp'
    create_repo(manifest_repo)

    # Commit west.yml describing projects and submodules dependencies.
    add_commit(manifest_repo, 'manifest repo commit',
               files={'west.yml':
                      f'''
                      manifest:
                        projects:
                        - name: zephyr
                          url: {zephyr}
                          submodules: true
                       '''})
    cmd(['init', '-l', manifest_repo])

    # Make tagged_repo to be zephyr project submodule.
    subprocess.check_call(SUBMODULE_ADD +
                          [str(tagged_repo), 'tagged_repo'],
                          cwd=zephyr)
    # Commit changes to the zephyr repo.
    add_commit(zephyr, 'zephyr submodule tagged_repo commit')

    # Make Kconfiglib to be net_tools submodule.
    subprocess.check_call(SUBMODULE_ADD +
                          [str(kconfiglib), 'Kconfiglib'],
                          cwd=net_tools)
    # Commit changes to the net_tools repo.
    add_commit(net_tools, 'net_tools submodule Kconfiglib commit')

    # Make net_tools to be zephyr project submodule.
    subprocess.check_call(SUBMODULE_ADD +
                          [str(net_tools), 'net-tools'],
                          cwd=zephyr)
    # Commit changes to the zephyr repo.
    add_commit(zephyr, 'zephyr submodule net-tools commit')

    # Get parsed data from the manifest.
    manifest = Manifest.from_topdir(topdir=ws,
                                    import_flags=MIF.IGNORE_PROJECTS)
    projects = manifest.get_projects(['zephyr'])
    zephyr_project = projects[0]

    # Verify if zephyr has submodules.
    assert zephyr_project.submodules

    # Check if project repo is cloned - should not be.
    assert not zephyr_project.is_cloned()

    # Update zephyr project.
    cmd(f'update {PROTOCOL_FILE_ALLOW} zephyr', cwd=ws)

    # Verify if zephyr project was cloned.
    assert zephyr_project.is_cloned()

    # Verify if tagged-repo submodule was also cloned.
    res = zephyr_project.git('rev-parse --show-cdup', check=False,
                             cwd=os.path.join(ws, 'zephyr', 'tagged_repo'),
                             capture_stderr=True, capture_stdout=True)
    assert not (res.returncode or res.stdout.strip())

    # Verify if net-tools submodule was also cloned.
    res = zephyr_project.git('rev-parse --show-cdup', check=False,
                             cwd=os.path.join(ws, 'zephyr', 'net-tools'),
                             capture_stderr=True, capture_stdout=True)
    assert not (res.returncode or res.stdout.strip())

    # Verify if Kconfiglib submodule was also cloned, as a result of recursive
    # update.
    res = zephyr_project.git('rev-parse --show-cdup', check=False,
                             cwd=os.path.join(ws, 'zephyr', 'net-tools',
                                              'Kconfiglib'),
                             capture_stderr=True, capture_stdout=True)
    assert not (res.returncode or res.stdout.strip())

    # Test freeze output with submodules
    # see test_manifest_freeze for details
    actual = cmd('manifest --freeze', cwd=ws).splitlines()
    expected_res = ['^manifest:$',
                    '^  projects:$',
                    '^  - name: zephyr$',
                    f'^    url: {re.escape(str(zephyr))}$',
                    '^    revision: [a-f0-9]{40}$',
                    '^    submodules: true$',
                    '^  self:$',
                    '^    path: mp$']
    _match_multiline_regex(expected_res, actual)

def test_update_no_submodules(repos_tmpdir):
    # Test verifies whether setting submodules value to boolean False does not
    # result in updating project submodules.

    # Repositories paths
    remotes = repos_tmpdir / 'repos'
    # tagged_repo and net_tools are zephyr's submodules
    zephyr = remotes / 'zephyr'
    tagged_repo = remotes / 'tagged_repo'
    net_tools = remotes / 'net-tools'

    # Creating west workspace and manifest repository.
    ws = repos_tmpdir / 'ws'
    create_workspace(ws)
    manifest_repo = ws / 'mp'
    create_repo(manifest_repo)

    # Commit west.yml describing projects and submodules dependencies.
    add_commit(manifest_repo, 'manifest repo commit',
               files={'west.yml':
                      f'''
                      manifest:
                        projects:
                        - name: zephyr
                          url: {zephyr}
                          submodules: false
                       '''})
    cmd(['init', '-l', manifest_repo])

    # Make tagged_repo to be zephyr project submodule.
    subprocess.check_call(SUBMODULE_ADD +
                          [str(tagged_repo), 'tagged_repo'],
                          cwd=zephyr)
    # Commit changes to the zephyr repo.
    add_commit(zephyr, 'zephyr submodule tagged_repo commit')

    # Make net_tools to be zephyr project submodule.
    subprocess.check_call(SUBMODULE_ADD +
                          [str(net_tools), 'net-tools'],
                          cwd=zephyr)
    # Commit changes to the zephyr repo.
    add_commit(zephyr, 'zephyr submodule net-tools commit')

    # Get parsed data from the manifest.
    manifest = Manifest.from_topdir(topdir=ws,
                                    import_flags=MIF.IGNORE_PROJECTS)
    projects = manifest.get_projects(['zephyr'])
    zephyr_project = projects[0]

    # Check if project repo is cloned - should not be.
    assert not zephyr_project.is_cloned()

    # Update zephyr project.
    cmd('update zephyr', cwd=ws)

    # Verify if zephyr project was cloned.
    assert zephyr_project.is_cloned()

    # Verify if tagged-repo submodule was also cloned (should not be).
    res = zephyr_project.git('rev-parse --show-cdup', check=False,
                             cwd=os.path.join(ws, 'zephyr', 'tagged_repo'),
                             capture_stderr=True, capture_stdout=True)
    assert (res.returncode or res.stdout.strip())

    # Verify if net-tools submodule was also cloned (should not be).
    res = zephyr_project.git('rev-parse --show-cdup', check=False,
                             cwd=os.path.join(ws, 'zephyr', 'net-tools'),
                             capture_stderr=True, capture_stdout=True)
    assert (res.returncode or res.stdout.strip())

    # Test freeze output with submodules
    # see test_manifest_freeze for details
    actual = cmd('manifest --freeze', cwd=ws).splitlines()
    expected_res = ['^manifest:$',
                    '^  projects:$',
                    '^  - name: zephyr$',
                    f'^    url: {re.escape(str(zephyr))}$',
                    '^    revision: [a-f0-9]{40}$',
                    '^  self:$',
                    '^    path: mp$']
    _match_multiline_regex(expected_res, actual)

def test_update_submodules_strategy(repos_tmpdir):
    # The west update command is able to update submodules using default
    # checkout strategy or rebase strategy, selected by adding -r argument
    # to the invoked command. Test verifies if both strategies are working
    # properly when updating submodules.

    # Repositories paths
    remotes = repos_tmpdir / 'repos'
    # tagged_repo is zephyr's submodule
    zephyr = remotes / 'zephyr'
    tagged_repo = remotes / 'tagged_repo'
    # Kconfiglib is net_tools's submodule
    net_tools = remotes / 'net-tools'
    kconfiglib = remotes / 'Kconfiglib'

    # Creating west workspace and manifest repository.
    ws = repos_tmpdir / 'ws'
    create_workspace(ws)
    manifest_repo = ws / 'mp'
    create_repo(manifest_repo)

    tagged_repo_dst_dir = os.path.join(ws, 'zephyr', 'tagged_repo')
    kconfiglib_dst_dir = os.path.join(ws, 'net-tools', 'Kconfiglib')

    # Commit west.yml describing projects and submodules dependencies.
    add_commit(manifest_repo, 'manifest repo commit',
               files={'west.yml':
                      f'''
                      manifest:
                        projects:
                        - name: zephyr
                          url: {zephyr}
                          submodules:
                            - name: tagged_repo
                              path: tagged_repo
                        - name: net-tools
                          url: {net_tools}
                          submodules:
                            - name: Kconfiglib
                              path: Kconfiglib
                       '''})
    cmd(['init', '-l', manifest_repo])

    # Make tagged_repo to be zephyr project submodule.
    subprocess.check_call(SUBMODULE_ADD +
                          [str(tagged_repo), 'tagged_repo'],
                          cwd=zephyr)
    # Commit changes to the zephyr repo.
    add_commit(zephyr, 'zephyr submodule change commit')

    # Make Kconfiglib to be net-tools project submodule.
    subprocess.check_call(SUBMODULE_ADD +
                          [str(kconfiglib), 'Kconfiglib'],
                          cwd=net_tools)
    # Commit changes to the net-tools repo.
    add_commit(net_tools, 'net-tools submodule change commit')

    # Get parsed data from the manifest.
    manifest = Manifest.from_topdir(topdir=ws,
                                    import_flags=MIF.IGNORE_PROJECTS)
    projects = manifest.get_projects(['zephyr', 'net-tools'])
    zephyr_project = projects[0]
    net_tools_project = projects[1]

    # Check if projects repos are cloned - should not be.
    assert not zephyr_project.is_cloned()
    assert not net_tools_project.is_cloned()

    # Update only zephyr project using checkout strategy (selected by default).
    cmd(f'update {PROTOCOL_FILE_ALLOW} zephyr', cwd=ws)

    # Verify if only zephyr project was cloned.
    assert zephyr_project.is_cloned()
    assert not net_tools_project.is_cloned()

    # Verify if tagged-repo submodule was also cloned
    res = zephyr_project.git('rev-parse --show-cdup', check=False,
                             cwd=tagged_repo_dst_dir, capture_stderr=True,
                             capture_stdout=True)
    assert not (res.returncode or res.stdout.strip())

    # Update only net-tools project using rebase strategy
    cmd(f'update {PROTOCOL_FILE_ALLOW} net-tools -r', cwd=ws)

    # Verify if both projects were cloned
    assert zephyr_project.is_cloned()
    assert net_tools_project.is_cloned()

    # Verify if Kconfiglib submodule was also cloned
    res = net_tools_project.git('rev-parse --show-cdup', check=False,
                                cwd=kconfiglib_dst_dir, capture_stderr=True,
                                capture_stdout=True)
    assert not (res.returncode or res.stdout.strip())

    # Save submodules HEAD revisions sha for verification purposes
    tagged_repo_head_sha = zephyr_project.sha('HEAD', cwd=tagged_repo_dst_dir)
    kconfiglib_head_sha = net_tools_project.sha('HEAD', cwd=kconfiglib_dst_dir)

    # Add commits to the submodules repos to modify their revisions
    add_commit(tagged_repo_dst_dir, 'tagged_repo test commit',
               files={'test.txt': "Test message"})
    add_commit(kconfiglib_dst_dir, 'Kconfiglib test commit',
               files={'test.txt': "Test message"})

    # Save submodules revisions sha after commit for verification purposes
    tagged_repo_new_sha = zephyr_project.sha('HEAD', cwd=tagged_repo_dst_dir)
    kconfiglib_new_sha = net_tools_project.sha('HEAD', cwd=kconfiglib_dst_dir)

    # Verify whether new revisions sha are different from the HEAD
    assert tagged_repo_head_sha != tagged_repo_new_sha
    assert kconfiglib_head_sha != kconfiglib_new_sha

    # Update only zephyr project using checkout strategy (selected by default).
    cmd('update zephyr', cwd=ws)

    # Verify if current submodule revision is HEAD, as checkout should drop
    # added commit.
    assert zephyr_project.sha('HEAD', cwd=tagged_repo_dst_dir) \
           == tagged_repo_head_sha

    # Update only net-tools project using rebase strategy
    cmd('update net-tools -r', cwd=ws)

    # Verify if current submodule revision is set to added commit, as rebase
    # should not drop it.
    assert net_tools_project.sha('HEAD', cwd=kconfiglib_dst_dir) \
           == kconfiglib_new_sha

    # Test freeze output with submodules
    # see test_manifest_freeze for details
    actual = cmd('manifest --freeze', cwd=ws).splitlines()
    expected_res = ['^manifest:$',
                    '^  projects:$',
                    '^  - name: zephyr$',
                    f'^    url: {re.escape(str(zephyr))}$',
                    '^    revision: [a-f0-9]{40}$',
                    '^    submodules:$',
                    '^    - path: tagged_repo$',
                    '^      name: tagged_repo$',
                    '^  - name: net-tools$',
                    f'^    url: {re.escape(str(net_tools))}$',
                    '^    revision: [a-f0-9]{40}$',
                    '^    submodules:$',
                    '^    - path: Kconfiglib$',
                    '^      name: Kconfiglib$',
                    '^  self:$',
                    '^    path: mp$']
    _match_multiline_regex(expected_res, actual)

@pytest.mark.xfail
def test_update_submodules_relpath(tmpdir):
    # Regression test for
    # https://github.com/zephyrproject-rtos/west/issues/545.
    #
    # We need to make sure that submodules with relative paths are
    # initialized correctly even when ther is no "origin" remote.
    #
    # Background: unless the config variable git.$SUBMODULENAME.url is
    # set, "git submodule init" relies on the "default remote" for
    # figuring out how to turn that relative path into something
    # useful. The "default remote" ends up being "origin" in the
    # situation where we are initializing a submodule from a
    # superproject which is a west project.
    #
    # However, west neither makes nor expects any guarantees
    # about any particular git remotes being present in a project, so
    # that only works if we set the config variable before
    # initializing the module.

    # The following paths begin with 'project' if they are west projects,
    # and 'submodule' if they are direct or transitive submodules of a
    # west project. Numeric suffixes reflect the tree structure of the
    # workspace:
    #
    # workspace
    # ├── manifest
    # ├── project-1
    # │   └── submodule-1-1
    # └── project-2
    #     ├── submodule-2-1
    #     └── submodule-2-2
    #         └── submodule-2-2-1
    #
    # Some of them are declared to west directly, some implicitly.
    pseudo_remotes = tmpdir / 'remotes'  # parent directory for 'remote' repos
    remote_repositories = [
        pseudo_remotes / path for path in
        ['manifest',
         'project-1',
         'submodule-1-1',
         'project-2',
         'submodule-2-1',
         'submodule-2-2',
         'submodule-2-2-1',
         ]
    ]
    for remote_repo in remote_repositories:
        create_repo(remote_repo)

    add_commit(pseudo_remotes / 'manifest', 'manifest west.yml',
               files={'west.yml': f'''
manifest:
  remotes:
    - name: not-origin
      url-base: file://{pseudo_remotes}
  defaults:
    remote: not-origin
  projects:
    - name: project-1
      submodules: true
    - name: project-2
      submodules:
        - path: submodule-2-1
        - name: sub-2-2
          path: submodule-2-2
'''})

    def add_submodule(superproject, submodule_name,
                      submodule_url, submodule_path):
        subprocess.check_call([GIT, '-C', os.fspath(superproject),
                               'submodule', 'add',
                               '--name', submodule_name,
                               submodule_url, submodule_path])
        add_commit(superproject, f'add submodule {submodule_name}')

    add_submodule(pseudo_remotes / 'project-1', 'sub-1-1',
                  '../submodule-1-1', 'submodule-1-1')
    add_submodule(pseudo_remotes / 'project-2', 'sub-2-1',
                  '../submodule-2-1', 'submodule-2-1')
    add_submodule(pseudo_remotes / 'project-2', 'sub-2-2',
                  '../submodule-2-2', 'submodule-2-2')
    add_submodule(pseudo_remotes / 'project-2' / 'submodule-2-2', 'sub-2-2-1',
                  '../submodule-2-2-1', 'submodule-2-2-1')

    workspace = tmpdir / 'workspace'
    remote_manifest = pseudo_remotes / 'manifest'
    cmd(f'init -m "{remote_manifest}" "{workspace}"')

    workspace.chdir()
    cmd('update')
    expected_dirs = [
        workspace / expected for expected in
        ['project-1',
         'project-1' / 'submodule-1-1',
         'project-2',
         'project-2' / 'submodule-2-1',
         'project-2' / 'submodule-2-2',
         'project-2' / 'submodule-2-2' / 'submodule-2-2-1',
         ]
    ]
    for expected_dir in expected_dirs:
        expected_dir.check(dir=1)

def test_update_recovery(tmpdir):
    # Make sure that the final 'west update' can recover from the
    # following turn of events:
    #
    #   1. 'm' is the manifest repository, 'p' is a project
    #   2. m/west.yml imports p at revision 'rbad'; p/west.yml at rbad
    #      contains an invalid manifest
    #   3. user runs 'west update', setting p's manifest-rev to rbad
    #      (and failing the update)
    #   4. user updates m/west.yml to point at p revision 'rgood',
    #      which contains good manifest data
    #   5. user runs 'west update' again
    #
    # The 'west update' in the last step should fix p's manifest-rev,
    # pointing it at rgood, and should succeed.

    # create path objects and string representations
    workspace = Path(tmpdir) / 'workspace'
    workspacestr = os.fspath(workspace)

    m = workspace / 'm'
    p = workspace / 'p'

    # Set up the workspace repositories.
    workspace.mkdir()
    create_repo(m)
    create_repo(p)

    # Create revision rbad, which contains a bogus manifest, in p.
    add_commit(p, 'rbad commit message', files={'west.yml': 'bogus_data'},
               reconfigure=False)
    rbad = rev_parse(p, 'HEAD')

    # Create revision rgood, which contains a good manifest, in p.
    add_commit(p, 'rgood commit message',
               files={'west.yml': 'manifest:\n  projects: []'},
               reconfigure=False)
    rgood = rev_parse(p, 'HEAD')

    # Set up the initial, 'bad' manifest.
    #
    # Use an invalid local file as the fetch URL: there's no reason
    # west should be fetching from the remote.
    with open(m / 'west.yml', 'w') as m_manifest:
        m_manifest.write(f'''
        manifest:
          projects:
          - name: p
            url: file://{tmpdir}/should-not-be-fetched
            revision: {rbad}
            import: true
        ''')

    # Use west init -l + west update to point p's manifest-rev at rbad.
    cmd(['init', '-l', m], cwd=workspacestr)
    with pytest.raises(subprocess.CalledProcessError):
        cmd('update', cwd=workspacestr)

    # Make sure p's manifest-rev points to the bad revision as expected.
    prev = rev_parse(p, 'refs/heads/manifest-rev')
    assert prev == rbad

    # Fix the main manifest to point at rgood.
    with open(m / 'west.yml', 'w') as m_manifest:
        m_manifest.write(f'''
        manifest:
          projects:
          - name: p
            url: file://{tmpdir}/should-not-be-fetched
            revision: {rgood}
            import: true
        ''')

    # Run the update, making sure it succeeds and p's manifest-rev
    # is fixed.
    cmd('update', cwd=workspacestr)
    prev = rev_parse(p, 'refs/heads/manifest-rev')
    assert prev == rgood


def setup_cache_workspace(workspace, remote, foo_head, bar_head):
    # Shared helper code that sets up a workspace used to test the
    # 'west update --foo-cache' options.

    create_workspace(workspace)

    manifest_project = workspace / 'mp'
    with open(manifest_project / 'west.yml', 'w') as f:
        f.write(f'''
        manifest:
          projects:
          - name: foo
            path: subdir/foo
            url: file://{remote}
            revision: {foo_head}
          - name: bar
            url: file://{remote}
            revision: {bar_head}
        ''')


def test_update_name_cache(tmpdir):
    # Test that 'west update --name-cache' works and doesn't hit the
    # network if it doesn't have to.

    remote = tmpdir / 'remote'
    create_repo(remote)
    name_cache_dir = tmpdir / 'name_cache'
    create_repo(name_cache_dir / 'foo')
    create_repo(name_cache_dir / 'bar')
    foo_head = rev_parse(name_cache_dir / 'foo', 'HEAD')
    bar_head = rev_parse(name_cache_dir / 'bar', 'HEAD')

    workspace = tmpdir / 'workspace'
    setup_cache_workspace(workspace, remote, foo_head, bar_head)
    workspace.chdir()
    foo = workspace / 'subdir' / 'foo'
    bar = workspace / 'bar'

    # Test the command line option.
    cmd(['update', '--name-cache', name_cache_dir])
    assert foo.check(dir=1)
    assert bar.check(dir=1)
    assert rev_parse(foo, 'HEAD') == foo_head
    assert rev_parse(bar, 'HEAD') == bar_head

    # Move the repositories out of the way and test the configuration option.
    # (We can't use shutil.rmtree here because Windows.)
    shutil.move(os.fspath(foo), os.fspath(tmpdir))
    shutil.move(os.fspath(bar), os.fspath(tmpdir))
    cmd(['config', 'update.name-cache', name_cache_dir])
    cmd('update')
    assert foo.check(dir=1)
    assert bar.check(dir=1)
    assert rev_parse(foo, 'HEAD') == foo_head
    assert rev_parse(bar, 'HEAD') == bar_head


def test_update_path_cache(tmpdir):
    # Test that 'west update --path-cache' works and doesn't hit the
    # network if it doesn't have to.

    remote = tmpdir / 'remote'
    create_repo(remote)
    path_cache_dir = tmpdir / 'path_cache_dir'
    create_repo(path_cache_dir / 'subdir' / 'foo')
    create_repo(path_cache_dir / 'bar')
    foo_head = rev_parse(path_cache_dir / 'subdir' / 'foo', 'HEAD')
    bar_head = rev_parse(path_cache_dir / 'bar', 'HEAD')

    workspace = tmpdir / 'workspace'
    setup_cache_workspace(workspace, remote, foo_head, bar_head)
    workspace.chdir()
    foo = workspace / 'subdir' / 'foo'
    bar = workspace / 'bar'

    # Test the command line option.
    cmd(['update', '--path-cache', path_cache_dir])
    assert foo.check(dir=1)
    assert bar.check(dir=1)
    assert rev_parse(foo, 'HEAD') == foo_head
    assert rev_parse(bar, 'HEAD') == bar_head

    # Move the repositories out of the way and test the configuration option.
    # (We can't use shutil.rmtree here because Windows.)
    shutil.move(os.fspath(foo), os.fspath(tmpdir))
    shutil.move(os.fspath(bar), os.fspath(tmpdir))
    cmd(['config', 'update.path-cache', path_cache_dir])
    cmd('update')
    assert foo.check(dir=1)
    assert bar.check(dir=1)
    assert rev_parse(foo, 'HEAD') == foo_head
    assert rev_parse(bar, 'HEAD') == bar_head


def setup_narrow(tmpdir):
    # Helper used by test_update_narrow() and test_update_narrow_depth1().

    remote = tmpdir / 'remote'

    create_repo(remote)
    add_commit(remote, 'second commit, main')
    add_tag(remote, 'tag')

    create_branch(remote, 'branch', checkout=True)
    add_commit(remote, 'second commit, branch', reconfigure=False)

    workspace = tmpdir / 'workspace'
    create_workspace(workspace)

    with open(workspace / 'mp' / 'west.yml', 'w') as f:
        f.write(f'''
        manifest:
          projects:
            - name: project
              revision: branch
              url: file://{remote}
        ''')

    return remote, workspace


def test_update_narrow(tmpdir):
    # Test that 'west update --narrow' doesn't fetch tags, and that
    # 'west update' respects the 'update.narrow' config option.

    remote, workspace = setup_narrow(tmpdir)
    workspace.chdir()

    def project_tags():
        return subprocess.check_output(
            [GIT, 'tag', '--list'], cwd=workspace / 'project'
        ).decode().splitlines()

    cmd('update --narrow')
    assert project_tags() == []

    cmd('config update.narrow true')
    cmd('update')
    assert project_tags() == []

    cmd('config update.narrow false')
    cmd('update')
    assert project_tags() != []


def test_update_narrow_depth1(tmpdir):
    # Test that 'west update --narrow -o=--depth=1' fetches exactly
    # one commit, regardless of how many there are in the remote
    # repository.

    remote, workspace = setup_narrow(tmpdir)

    cmd('update --narrow --fetch-opt=--depth=1', cwd=workspace)

    refs = subprocess.check_output(
        [GIT, 'for-each-ref'], cwd=workspace / 'project',
    ).decode().splitlines()

    assert len(refs) == 1


def test_init_again(west_init_tmpdir):
    # Test that 'west init' on an initialized tmpdir errors out
    # with a message that indicates it's already initialized.

    popen = subprocess.Popen('west init'.split(),
                             stdout=subprocess.DEVNULL,
                             stderr=subprocess.PIPE,
                             cwd=west_init_tmpdir)
    _, stderr = popen.communicate()
    assert popen.returncode
    assert b'already initialized' in stderr

    popen = subprocess.Popen('west init -m http://example.com'.split(),
                             stdout=subprocess.DEVNULL,
                             stderr=subprocess.PIPE,
                             cwd=west_init_tmpdir)
    _, stderr = popen.communicate()
    assert popen.returncode
    assert b'already initialized' in stderr

    manifest = west_init_tmpdir / '..' / 'repos' / 'zephyr'
    popen = subprocess.Popen(
        ['west', '-vvv', 'init', '-m', str(manifest), 'workspace'],
        stdout=subprocess.DEVNULL,
        stderr=subprocess.PIPE,
        cwd=west_init_tmpdir.dirname)
    _, stderr = popen.communicate()
    assert popen.returncode
    assert b'already initialized' in stderr

def test_init_local_manifest_project(repos_tmpdir):
    # Do a local clone of manifest repo
    zephyr_install_dir = repos_tmpdir.join('workspace', 'zephyr')
    clone(str(repos_tmpdir.join('repos', 'zephyr')),
          str(zephyr_install_dir))

    cmd(['init', '-l', zephyr_install_dir])

    # Verify Zephyr has been installed during init -l, but not projects.
    zid = repos_tmpdir.join('workspace')
    assert zid.check(dir=1)
    assert zid.join('subdir', 'Kconfiglib').check(dir=0)
    assert zid.join('net-tools').check(dir=0)
    assert zid.join('tagged_repo').check(dir=0)
    assert zid.join('zephyr').check(dir=1)
    assert zid.join('zephyr', '.git').check(dir=1)
    assert zid.join('zephyr', 'CODEOWNERS').check(file=1)
    assert zid.join('zephyr', 'include', 'header.h').check(file=1)
    assert zid.join('zephyr', 'subsys', 'bluetooth', 'code.c').check(file=1)

    cmd('update', cwd=str(zid))
    # The projects should be installled now
    assert zid.check(dir=1)
    assert zid.join('subdir', 'Kconfiglib').check(dir=1)
    assert zid.join('net-tools').check(dir=1)
    assert zid.join('tagged_repo').check(dir=1)
    assert zid.join('subdir', 'Kconfiglib').check(dir=1)
    assert zid.join('subdir', 'Kconfiglib', '.git').check(dir=1)
    assert zid.join('subdir', 'Kconfiglib', 'kconfiglib.py').check(file=1)
    assert zid.join('net-tools').check(dir=1)
    assert zid.join('net-tools', '.git').check(dir=1)
    assert zid.join('net-tools', 'qemu-script.sh').check(file=1)
    assert zid.join('tagged_repo', 'test.txt').check(file=1)


def test_init_local_already_initialized_failure(west_init_tmpdir):
    # Test that 'west init -l' on an initialized tmpdir errors out
    with pytest.raises(subprocess.CalledProcessError):
        cmd(f'init -l "{west_init_tmpdir}"')


def test_init_local_missing_west_yml_failure(repos_tmpdir):
    # Test that 'west init -l' on repo without a 'west.yml' fails

    # Do a local clone of manifest repo
    zephyr_install_dir = repos_tmpdir.join('workspace', 'zephyr')
    clone(str(repos_tmpdir.join('repos', 'zephyr')),
          str(zephyr_install_dir))
    os.remove(str(zephyr_install_dir.join('west.yml')))

    with pytest.raises(subprocess.CalledProcessError):
        cmd(f'init -l "{zephyr_install_dir}"')


def test_init_local_with_manifest_filename(repos_tmpdir):
    # Test 'west init --mf -l' on a local repo

    manifest = repos_tmpdir / 'repos' / 'zephyr'
    workspace = repos_tmpdir / 'workspace'
    zephyr_install_dir = workspace / 'zephyr'

    # Do a local clone of manifest repo
    clone(str(manifest), str(zephyr_install_dir))
    os.rename(str(zephyr_install_dir / 'west.yml'),
              str(zephyr_install_dir / 'project.yml'))

    # fails because west.yml is missing
    with pytest.raises(subprocess.CalledProcessError):
        cmd(['init', '-l', zephyr_install_dir])

    # create a manifest with a syntax error so we can test if it's being parsed
    with open(zephyr_install_dir / 'west.yml', 'w') as f:
        f.write('[')

    cwd = os.getcwd()
    cmd(['init', '-l', zephyr_install_dir])

    # init with a local manifest doesn't parse the file, so let's access it
    workspace.chdir()
    with pytest.raises(subprocess.CalledProcessError):
        cmd('list')

    os.chdir(cwd)
    shutil.move(workspace / '.west', workspace / '.west-syntaxerror')

    # success
    cmd(['init', '--mf', 'project.yml', '-l', zephyr_install_dir])
    workspace.chdir()
    cmd('update')


def test_init_local_with_empty_path(repos_tmpdir):
    # Test "west init -l ." + "west update".
    # Regression test for:
    # https://github.com/zephyrproject-rtos/west/issues/435

    local_manifest_dir = repos_tmpdir / 'workspace' / 'zephyr'
    clone(str(repos_tmpdir / 'repos' / 'zephyr'), str(local_manifest_dir))
    os.chdir(local_manifest_dir)
    cmd('init -l .')
    cmd('update')
    assert (repos_tmpdir / 'workspace' / 'subdir' / 'Kconfiglib').check(dir=1)


def test_init_local_with_clone_option_failure(repos_tmpdir):
    # Test that 'west init -l -o' errors out

    west_tmpdir = repos_tmpdir / 'workspace'

    with pytest.raises(subprocess.CalledProcessError):
        cmd(['init', '-l', '-o=--depth=1', west_tmpdir])


def test_init_with_clone_option_depth_one(repos_tmpdir):
    # Test that 'west init -o=--depth=1' only clones depth 1

    west_tmpdir = repos_tmpdir / 'workspace'

    cmd(['init', '-o=--depth=1', west_tmpdir])
    assert 1 == int(subprocess.check_output([GIT, 'rev-list', '--count', '--max-count=5', 'HEAD'],
                                            cwd=west_tmpdir / 'zephyr').decode().strip())


def test_update_with_groups_enabled(west_init_tmpdir):
    # Test "west update" with increasing numbers of groups enabled.

    remotes = west_init_tmpdir / '..' / 'repos'

    with open(west_init_tmpdir / 'zephyr' / 'west.yml', 'w') as f:
        # The purpose of the 'disabled' group is to ensure that a
        # project's groups' "enabled" bits are ORed together, not
        # ANDed together, when deciding if the project is active.
        f.write(f'''
        manifest:
          defaults:
            remote: test-local
          remotes:
            - name: test-local
              url-base: {remotes}
          projects:
            - name: Kconfiglib
              revision: zephyr
              path: subdir/Kconfiglib
              groups:
              - enabled
              - disabled
            - name: tagged_repo
              revision: v1.0
              groups:
              - enable-on-cmd-line
              - disabled
            - name: net-tools
              groups:
              - enable-in-config-file
              - disabled
          group-filter: [-enable-on-cmd-line,-enable-in-config-file,-disabled]
          self:
            path: zephyr
        ''')

    cmd('update')
    assert (west_init_tmpdir / 'subdir' / 'Kconfiglib').check(dir=1)
    assert (west_init_tmpdir / 'tagged_repo').check(dir=0)
    assert (west_init_tmpdir / 'net-tools').check(dir=0)

    cmd('update --group-filter +enable-on-cmd-line')
    assert (west_init_tmpdir / 'tagged_repo').check(dir=1)
    assert (west_init_tmpdir / 'net-tools').check(dir=0)

    cmd('config manifest.group-filter +enable-in-config-file')
    cmd('update')
    assert (west_init_tmpdir / 'net-tools').check(dir=1)


def test_update_with_groups_disabled(west_init_tmpdir):
    # Test "west update" with decreasing numbers of groups disabled.

    remotes = west_init_tmpdir / '..' / 'repos'

    with open(west_init_tmpdir / 'zephyr' / 'west.yml', 'w') as f:
        f.write(f'''
        manifest:
          defaults:
            remote: test-local
          remotes:
            - name: test-local
              url-base: {remotes}
          projects:
            - name: Kconfiglib
              revision: zephyr
              path: subdir/Kconfiglib
              groups:
              - disabled
            - name: tagged_repo
              revision: v1.0
              groups:
              - disabled-on-cmd-line
            - name: net-tools
              groups:
              - disabled-in-config-file
          group-filter: [-disabled]
          self:
            path: zephyr
        ''')

    cmd('config manifest.group-filter -- -disabled-in-config-file')
    cmd('update --group-filter=-disabled-on-cmd-line')
    assert (west_init_tmpdir / 'subdir' / 'Kconfiglib').check(dir=0)
    assert (west_init_tmpdir / 'tagged_repo').check(dir=0)
    assert (west_init_tmpdir / 'net-tools').check(dir=0)

    cmd('config -d manifest.group-filter')
    cmd('update --group-filter=-disabled-on-cmd-line')
    assert (west_init_tmpdir / 'subdir' / 'Kconfiglib').check(dir=0)
    assert (west_init_tmpdir / 'tagged_repo').check(dir=0)
    assert (west_init_tmpdir / 'net-tools').check(dir=1)

    cmd('update')
    assert (west_init_tmpdir / 'subdir' / 'Kconfiglib').check(dir=0)
    assert (west_init_tmpdir / 'tagged_repo').check(dir=1)

    # Enabling overrides disabling.
    cmd('update --group-filter +disabled')
    assert (west_init_tmpdir / 'subdir' / 'Kconfiglib').check(dir=1)

def test_update_with_groups_explicit(west_init_tmpdir):
    # Even inactive projects must be updated if the user specifically
    # requests it.

    remotes = west_init_tmpdir / '..' / 'repos'

    with open(west_init_tmpdir / 'zephyr' / 'west.yml', 'w') as f:
        f.write(f'''
        manifest:
          defaults:
            remote: test-local
          remotes:
            - name: test-local
              url-base: {remotes}
          projects:
            - name: Kconfiglib
              revision: zephyr
              path: subdir/Kconfiglib
              groups:
              - disabled
          group-filter: [-disabled]
          self:
            path: zephyr
        ''')

    cmd('update')
    assert (west_init_tmpdir / 'subdir' / 'Kconfiglib').check(dir=0)

    cmd('update Kconfiglib')
    assert (west_init_tmpdir / 'subdir' / 'Kconfiglib').check(dir=1)


def test_init_with_manifest_filename(repos_tmpdir):
    # Test 'west init --mf' on a normal repo

    west_tmpdir = repos_tmpdir / 'workspace'
    manifest = repos_tmpdir / 'repos' / 'zephyr'

    with open(manifest / 'west.yml') as f:
        manifest_data = f.read()

    # also creates a west.yml with a syntax error to verify west doesn't even
    # try to load the file
    add_commit(str(manifest), 'rename manifest',
               files={'west.yml': '[', 'project.yml': manifest_data})

    # syntax error
    with pytest.raises(subprocess.CalledProcessError):
        cmd(['init', '-m', manifest, west_tmpdir])
    shutil.move(west_tmpdir, repos_tmpdir / 'workspace-syntaxerror')

    # success
    cmd(['init', '-m', manifest, '--mf', 'project.yml', west_tmpdir])
    west_tmpdir.chdir()
    cmd('update')

def test_init_with_manifest_in_subdir(repos_tmpdir):
    # Test west init with a manifest repository that is intended to
    # live in a nested subdirectory of the workspace topdir.

    manifest = repos_tmpdir / 'repos' / 'zephyr'
    add_commit(manifest, 'move manifest repo to subdirectory',
               files={'west.yml':
                      textwrap.dedent('''
                      manifest:
                        projects: []
                        self:
                          path: nested/subdirectory
                      ''')})

    workspace = repos_tmpdir / 'workspace'
    cmd(['init', '-m', manifest, workspace])
    assert (workspace / 'nested' / 'subdirectory').check(dir=1)

def test_extension_command_execution(west_init_tmpdir):
    with pytest.raises(subprocess.CalledProcessError):
        cmd('test-extension')

    cmd('update')

    actual = cmd('test-extension')
    assert actual.rstrip() == 'Testing test command 1'


def test_extension_command_multiproject(repos_tmpdir):
    # Test to ensure that multiple projects can define extension commands and
    # that those are correctly presented and executed.
    rr = repos_tmpdir.join('repos')
    remote_kconfiglib = str(rr.join('Kconfiglib'))
    remote_zephyr = str(rr.join('zephyr'))
    remote_west = str(rr.join('west'))

    # Update the manifest to specify extension commands in Kconfiglib.
    # This removes tagged_repo, but we're not using it, so that's fine.
    add_commit(remote_zephyr, 'test added extension command',
               files={'west.yml': textwrap.dedent(f'''\
                      west:
                        url: file://{remote_west}
                      manifest:
                        defaults:
                          remote: test-local

                        remotes:
                          - name: test-local
                            url-base: file://{rr}

                        projects:
                          - name: Kconfiglib
                            revision: zephyr
                            path: subdir/Kconfiglib
                            west-commands: scripts/west-commands.yml
                          - name: net-tools
                            west-commands: scripts/west-commands.yml
                        self:
                          path: zephyr
                      ''')})

    # Add an extension command to the Kconfiglib remote.
    add_commit(remote_kconfiglib, 'add west commands',
               files={'scripts/west-commands.yml': textwrap.dedent('''\
                      west-commands:
                        - file: scripts/test.py
                          commands:
                            - name: kconfigtest
                              class: Test
                      '''),
                      'scripts/test.py': textwrap.dedent('''\
                      from west.commands import WestCommand
                      class Test(WestCommand):
                          def __init__(self):
                              super(Test, self).__init__(
                                  'kconfigtest',
                                  'Kconfig test application',
                                  '')
                          def do_add_parser(self, parser_adder):
                              parser = parser_adder.add_parser(self.name)
                              return parser
                          def do_run(self, args, ignored):
                              print('Testing kconfig test')
                      '''),
                      })
    west_tmpdir = repos_tmpdir / 'workspace'
    zephyr = repos_tmpdir / 'repos' / 'zephyr'
    cmd(['init', '-m', zephyr, west_tmpdir])
    west_tmpdir.chdir()
    cmd('update')

    # The newline shenanigans are for Windows.
    help_text = '\n'.join(cmd('-h').splitlines())
    expected = '\n'.join([
        'extension commands from project Kconfiglib (path: subdir/Kconfiglib):',  # noqa: E501
        '  kconfigtest:          (no help provided; try "west kconfigtest -h")',  # noqa: E501
        '',
        'extension commands from project net-tools (path: net-tools):',
        '  test-extension:       test-extension-help'])
    assert expected in help_text, help_text

    actual = cmd('test-extension')
    assert actual.rstrip() == 'Testing test command 1'

    actual = cmd('kconfigtest')
    assert actual.rstrip() == 'Testing kconfig test'


def test_extension_command_duplicate(repos_tmpdir):
    # West should disregard an extension command if its name is already taken,
    # either by a built-in command or another extension command added earlier.
    # A warning should also appear for each such case.

    rr = repos_tmpdir.join('repos')
    remote_kconfiglib = str(rr.join('Kconfiglib'))
    remote_zephyr = str(rr.join('zephyr'))
    remote_west = str(rr.join('west'))

    # This removes tagged_repo, but we're not using it, so that's fine.
    add_commit(remote_zephyr, 'test added extension command',
               files={'west.yml': textwrap.dedent(f'''\
                      west:
                        url: file://{remote_west}
                      manifest:
                        defaults:
                          remote: test-local

                        remotes:
                          - name: test-local
                            url-base: file://{rr}

                        projects:
                          - name: Kconfiglib
                            revision: zephyr
                            path: subdir/Kconfiglib
                            west-commands: scripts/west-commands.yml
                          - name: net-tools
                            west-commands: scripts/west-commands.yml
                        self:
                          path: zephyr
                      ''')})

    # Add extension commands to the Kconfiglib remote.
    add_commit(remote_kconfiglib, 'add west commands',
               files={'scripts/west-commands.yml': textwrap.dedent('''\
                      west-commands:
                        - file: scripts/test.py
                          commands:
                            - name: list
                              class: List
                            - name: test-extension
                              class: Test
                      '''),
                      'scripts/test.py': textwrap.dedent('''\
                      from argparse import REMAINDER
                      from west.commands import WestCommand
                      class List(WestCommand):
                          def __init__(self):
                              super(List, self).__init__(
                                  'list',
                                  'test list',
                                  '')
                          def do_add_parser(self, parser_adder):
                              parser = parser_adder.add_parser(self.name)
                              parser.add_argument('any', nargs=REMAINDER)
                              return parser
                          def do_run(self, args, ignored):
                              print('This must never be printed')
                      class Test(WestCommand):
                          def __init__(self):
                              super(Test, self).__init__(
                                  'test-extension',
                                  'test application',
                                  '')
                          def do_add_parser(self, parser_adder):
                              parser = parser_adder.add_parser(self.name)
                              return parser
                          def do_run(self, args, ignored):
                              print('Testing kconfig test command')
                      '''),
                      })
    west_tmpdir = repos_tmpdir / 'workspace'
    zephyr = repos_tmpdir / 'repos' / 'zephyr'
    cmd(['init', '-m', zephyr, west_tmpdir])
    west_tmpdir.chdir()
    cmd('update')

    expected_warns = [
        'WARNING: ignoring project Kconfiglib extension command "list"; '
        'this is a built in command',
        'WARNING: ignoring project net-tools extension command "test-extension"; '
        'command "test-extension" is already defined as extension command',
    ]

    # Expect output from the built-in command, not its Kconfiglib duplicate.
    actual = cmd('list zephyr -f {name}', stderr=subprocess.STDOUT).splitlines()
    assert actual == expected_warns + ['manifest']

    # Expect output from the Kconfiglib command, not its net-tools duplicate.
    actual = cmd('test-extension', stderr=subprocess.STDOUT).splitlines()
    assert actual == expected_warns + ['Testing kconfig test command']

def test_topdir_none(tmpdir):
    # Running west topdir outside of any workspace ought to fail.

    tmpdir.chdir()
    with pytest.raises(subprocess.CalledProcessError):
        cmd('topdir')

def test_topdir_in_workspace(west_init_tmpdir):
    # Running west topdir anywhere inside of a workspace ought to
    # work, and return the same thing.

    expected = PurePath(str(west_init_tmpdir)).as_posix()

    # This should be available immediately after west init.
    assert cmd('topdir').strip() == expected

    # After west update, it should continue to work, and return the
    # same thing (not getting confused when called from within a
    # project directory or a random user-created subdirectory, e.g.)
    cmd('update')
    assert cmd('topdir', cwd=str(west_init_tmpdir / 'subdir' /
                                 'Kconfiglib')).strip() == expected
    west_init_tmpdir.mkdir('pytest-foo')
    assert cmd('topdir', cwd=str(west_init_tmpdir /
                                 'pytest-foo')).strip() == expected

#
# Helper functions used by the test cases and fixtures.
#


def clone(repo, dst):
    # Creates a new branch.
    repo = str(repo)

    subprocess.check_call([GIT, 'clone', repo, dst])


def checkout_branch(repo, branch, create=False):
    # Creates a new branch.
    repo = str(repo)

    # Edit any files as specified by the user and add them to the index.
    if create:
        subprocess.check_call([GIT, 'checkout', '-b', branch], cwd=repo)
    else:
        subprocess.check_call([GIT, 'checkout', branch], cwd=repo)


def head_subject(path):
    # Returns the subject of the HEAD commit in the repository at 'path'

    return subprocess.check_output([GIT, 'log', '-n1', '--format=%s'],
                                   cwd=path).decode().rstrip()

def default_updater(remotes):
    add_commit(remotes.net_tools, 'another net-tools commit')
    add_commit(remotes.kconfiglib, 'another kconfiglib commit')
    add_commit(remotes.tagged_repo, 'another tagged_repo commit')
    cmd('update')

def update_helper(west_tmpdir, updater=default_updater):
    # Helper command for causing a change in two remote repositories,
    # then running a project command on the west workspace.
    #
    # Adds a commit to both of the kconfiglib and net-tools projects
    # remotes, then call updater(update_remotes),
    # which defaults to a function that adds commits in each
    # repository's remote and runs 'west update'.
    #
    # Captures the remote and local repository paths, as well as
    # manifest-rev and HEAD before and after, returning the results in
    # an UpdateResults tuple.

    nt_remote = str(west_tmpdir.join('..', 'repos', 'net-tools'))
    nt_local = str(west_tmpdir.join('net-tools'))
    kl_remote = str(west_tmpdir.join('..', 'repos', 'Kconfiglib'))
    kl_local = str(west_tmpdir.join('subdir', 'Kconfiglib'))
    tr_remote = str(west_tmpdir.join('..', 'repos', 'tagged_repo'))
    tr_local = str(west_tmpdir.join('tagged_repo'))

    def output_or_none(*args, **kwargs):
        try:
            ret = check_output(*args, **kwargs)
        except (FileNotFoundError, NotADirectoryError,
                subprocess.CalledProcessError):
            ret = None
        return ret

    nt_mr_0 = output_or_none([GIT, 'rev-parse', 'manifest-rev'], cwd=nt_local)
    kl_mr_0 = output_or_none([GIT, 'rev-parse', 'manifest-rev'], cwd=kl_local)
    tr_mr_0 = output_or_none([GIT, 'rev-parse', 'manifest-rev'], cwd=tr_local)
    nt_head_0 = output_or_none([GIT, 'rev-parse', 'HEAD'], cwd=nt_local)
    kl_head_0 = output_or_none([GIT, 'rev-parse', 'HEAD'], cwd=kl_local)
    tr_head_0 = output_or_none([GIT, 'rev-parse', 'HEAD'], cwd=tr_local)

    updater(UpdateRemotes(nt_remote, kl_remote, tr_remote))

    nt_mr_1 = output_or_none([GIT, 'rev-parse', 'manifest-rev'], cwd=nt_local)
    kl_mr_1 = output_or_none([GIT, 'rev-parse', 'manifest-rev'], cwd=kl_local)
    tr_mr_1 = output_or_none([GIT, 'rev-parse', 'manifest-rev'], cwd=tr_local)
    nt_head_1 = output_or_none([GIT, 'rev-parse', 'HEAD'], cwd=nt_local)
    kl_head_1 = output_or_none([GIT, 'rev-parse', 'HEAD'], cwd=kl_local)
    tr_head_1 = output_or_none([GIT, 'rev-parse', 'HEAD'], cwd=tr_local)

    return UpdateResults(nt_remote, nt_local,
                         kl_remote, kl_local,
                         tr_remote, tr_local,
                         nt_mr_0, nt_mr_1,
                         kl_mr_0, kl_mr_1,
                         tr_mr_0, tr_mr_1,
                         nt_head_0, nt_head_1,
                         kl_head_0, kl_head_1,
                         tr_head_0, tr_head_1)

def test_change_remote_conflict(west_update_tmpdir):
    # Test that `west update` will force fetch into local refs space when
    # remote has changed and cannot be fast forwarded.
    wct = west_update_tmpdir
    tmpdir = wct.join('..')

    rrepo = str(tmpdir.join('repos'))
    net_tools = str(tmpdir.join('repos', 'net-tools'))
    rwest = str(tmpdir.join('repos', 'west'))
    alt_repo = str(tmpdir.join('alt_repo'))
    alt_net_tools = str(tmpdir.join('alt_repo', 'net-tools'))
    create_repo(alt_net_tools)
    add_commit(alt_net_tools, 'test conflicting commit',
               files={'qemu-script.sh': 'echo alternate world net-tools\n'})

    revision = rev_parse(net_tools, 'HEAD')

    west_yml_content = textwrap.dedent(f'''\
                      west:
                        url: file://{rwest}
                      manifest:
                        defaults:
                          remote: test-local

                        remotes:
                          - name: test-local
                            url-base: file://{rrepo}

                        projects:
                          - name: net-tools
                            revision: {revision}
                        self:
                          path: zephyr
                      ''')
    add_commit(str(wct.join('zephyr')), 'test update manifest',
               files={'west.yml': west_yml_content})

    cmd('update')

    revision = rev_parse(alt_net_tools, 'HEAD')

    west_yml_content = textwrap.dedent(f'''\
                      west:
                        url: file://{rwest}
                      manifest:
                        defaults:
                          remote: test-local

                        remotes:
                          - name: test-local
                            url-base: file://{rrepo}
                          - name: test-alternate
                            url-base: file://{alt_repo}

                        projects:
                          - name: net-tools
                            remote: test-alternate
                            revision: {revision}
                        self:
                          path: zephyr
                      ''')

    add_commit(str(wct.join('zephyr')), 'test update manifest conflict',
               files={'west.yml': west_yml_content})

    cmd('update')

def test_import_project_release(repos_tmpdir):
    # Tests for a workspace that's based off of importing from a
    # project at a fixed release, with no downstream project forks.

    remotes = repos_tmpdir / 'repos'
    zephyr = remotes / 'zephyr'
    add_tag(zephyr, 'test-tag')

    # For this test, we create a remote manifest repository. This
    # makes sure we can clone a manifest repository which contains
    # imports without issue (and we don't need to clone the imported
    # projects.)
    #
    # On subsequent tests, we won't bother with this step. We will
    # just put the manifest repository directly into the workspace and
    # use west init -l. This also provides coverage for the -l option
    # in the presence of imports.
    manifest_remote = remotes / 'mp'
    create_repo(manifest_remote)
    add_commit(manifest_remote, 'manifest repo commit',
               files={'west.yml':
                      f'''
                      manifest:
                        projects:
                        - name: zephyr
                          url: {zephyr}
                          revision: test-tag
                          import: true
                      '''})

    # Create the workspace and verify we can't load the manifest yet
    # (because some imported data is missing).
    ws = repos_tmpdir / 'ws'
    cmd(['init', '-m', manifest_remote, ws])
    with pytest.raises(ManifestImportFailed):
        # We can't load this yet, because we haven't cloned zephyr.
        Manifest.from_topdir(topdir=ws)

    # Run west update and make sure we can load the manifest now.
    cmd('update', cwd=ws)

    actual = Manifest.from_topdir(topdir=ws).projects
    expected = [ManifestProject(path='mp', topdir=ws),
                Project('zephyr', zephyr,
                        revision='test-tag', topdir=ws),
                Project('Kconfiglib', remotes / 'Kconfiglib',
                        revision='zephyr', path='subdir/Kconfiglib',
                        topdir=ws),
                Project('tagged_repo', remotes / 'tagged_repo',
                        revision='v1.0', topdir=ws),
                Project('net-tools', remotes / 'net-tools',
                        clone_depth=1, topdir=ws,
                        west_commands='scripts/west-commands.yml')]
    for a, e in zip(actual, expected):
        check_proj_consistency(a, e)

    # Add a commit in the remote zephyr repository and make sure it
    # doesn't affect our local workspace, since we've locked it to a
    # tag.
    zephyr_ws = ws / 'zephyr'
    head_before = rev_parse(zephyr_ws, 'HEAD')
    add_commit(zephyr, 'this better not show up',
               files={'should-not-clone': ''})

    cmd('update', cwd=ws)

    assert head_before == rev_parse(zephyr_ws, 'HEAD')
    actual = Manifest.from_topdir(topdir=ws).projects
    for a, e in zip(actual, expected):
        check_proj_consistency(a, e)
    assert (zephyr_ws / 'should-not-clone').check(file=0)

def test_import_project_release_fork(repos_tmpdir):
    # Like test_import_project_release(), but with a project fork,
    # and using west init -l.

    remotes = repos_tmpdir / 'repos'

    zephyr = remotes / 'zephyr'
    add_tag(zephyr, 'zephyr-tag')

    fork = remotes / 'my-kconfiglib-fork'
    create_repo(fork)
    add_commit(fork, 'fork kconfiglib')
    add_tag(fork, 'fork-tag')

    ws = repos_tmpdir / 'ws'
    create_workspace(ws)
    manifest_repo = ws / 'mp'
    create_repo(manifest_repo)
    add_commit(manifest_repo, 'manifest repo commit',
               files={'west.yml':
                      f'''
                      manifest:
                        projects:
                        - name: zephyr
                          url: {zephyr}
                          revision: zephyr-tag
                          import: true
                        - name: Kconfiglib
                          url: {fork}
                          revision: fork-tag
                      '''})

    cmd(['init', '-l', manifest_repo])
    with pytest.raises(ManifestImportFailed):
        Manifest.from_topdir(topdir=ws)

    cmd('update', cwd=ws)

    actual = Manifest.from_topdir(topdir=ws).projects
    expected = [ManifestProject(path='mp', topdir=ws),
                Project('zephyr', zephyr,
                        revision='zephyr-tag', topdir=ws),
                Project('Kconfiglib', fork,
                        revision='fork-tag', path='Kconfiglib',
                        topdir=ws),
                Project('tagged_repo', remotes / 'tagged_repo',
                        revision='v1.0', topdir=ws),
                Project('net-tools', remotes / 'net-tools',
                        clone_depth=1, topdir=ws,
                        west_commands='scripts/west-commands.yml')]
    for a, e in zip(actual, expected):
        check_proj_consistency(a, e)

    zephyr_ws = ws / 'zephyr'
    head_before = rev_parse(zephyr_ws, 'HEAD')
    add_commit(zephyr, 'this better not show up',
               files={'should-not-clone': ''})

    cmd('update', cwd=ws)

    assert head_before == rev_parse(zephyr_ws, 'HEAD')
    actual = Manifest.from_topdir(topdir=ws).projects
    for a, e in zip(actual, expected):
        check_proj_consistency(a, e)
    assert (zephyr_ws / 'should-not-clone').check(file=0)

def test_import_project_release_dir(tmpdir):
    # Tests for a workspace that imports a directory from a project
    # at a fixed release.

    remotes = tmpdir / 'remotes'
    empty_project = remotes / 'empty_project'
    create_repo(empty_project)
    add_commit(empty_project, 'empty-project empty commit')
    imported = remotes / 'imported'
    create_repo(imported)
    add_commit(imported, 'add directory of imports',
               files={'test.d/1.yml':
                      f'''\
                      manifest:
                        projects:
                        - name: west.d_1.yml-p1
                          url: {empty_project}
                        - name: west.d_1.yml-p2
                          url: {empty_project}
                      ''',
                      'test.d/2.yml':
                      f'''\
                      manifest:
                        projects:
                        - name: west.d_2.yml-p1
                          url: {empty_project}
                      '''})
    add_tag(imported, 'import-tag')

    ws = tmpdir / 'ws'
    create_workspace(ws)
    manifest_repo = ws / 'mp'
    add_commit(manifest_repo, 'manifest repo commit',
               files={'west.yml':
                      f'''
                      manifest:
                        projects:
                        - name: imported
                          url: {imported}
                          revision: import-tag
                          import: test.d
                      '''})

    cmd(['init', '-l', manifest_repo])
    with pytest.raises(ManifestImportFailed):
        Manifest.from_topdir(topdir=ws)

    cmd('update', cwd=ws)
    actual = Manifest.from_topdir(topdir=ws).projects
    expected = [ManifestProject(path='mp', topdir=ws),
                Project('imported', imported,
                        revision='import-tag', topdir=ws),
                Project('west.d_1.yml-p1', empty_project, topdir=ws),
                Project('west.d_1.yml-p2', empty_project, topdir=ws),
                Project('west.d_2.yml-p1', empty_project, topdir=ws)]
    for a, e in zip(actual, expected):
        check_proj_consistency(a, e)

def test_import_project_rolling(repos_tmpdir):
    # Like test_import_project_release, but with a rolling downstream
    # that pulls master. We also use west init -l.

    remotes = repos_tmpdir / 'repos'
    zephyr = remotes / 'zephyr'

    ws = repos_tmpdir / 'ws'
    create_workspace(ws)
    manifest_repo = ws / 'mp'
    create_repo(manifest_repo)
    add_commit(manifest_repo, 'manifest repo commit',
               # zephyr revision is implicitly master:
               files={'west.yml':
                      f'''
                      manifest:
                        projects:
                        - name: zephyr
                          url: {zephyr}
                          import: true
                      '''})

    cmd(['init', '-l', manifest_repo])
    with pytest.raises(ManifestImportFailed):
        Manifest.from_topdir(topdir=ws)

    cmd('update', cwd=ws)

    actual = Manifest.from_topdir(topdir=ws).projects
    expected = [ManifestProject(path='mp', topdir=ws),
                Project('zephyr', zephyr,
                        revision='master', topdir=ws),
                Project('Kconfiglib', remotes / 'Kconfiglib',
                        revision='zephyr', path='subdir/Kconfiglib',
                        topdir=ws),
                Project('tagged_repo', remotes / 'tagged_repo',
                        revision='v1.0', topdir=ws),
                Project('net-tools', remotes / 'net-tools',
                        clone_depth=1, topdir=ws,
                        west_commands='scripts/west-commands.yml')]
    for a, e in zip(actual, expected):
        check_proj_consistency(a, e)

    # Add a commit in the remote zephyr repository and make sure it
    # *does* affect our local workspace, since we're rolling with its
    # master branch.
    zephyr_ws = ws / 'zephyr'
    head_before = rev_parse(zephyr_ws, 'HEAD')
    add_commit(zephyr, 'this better show up',
               files={'should-clone': ''})

    cmd('update', cwd=ws)

    assert head_before != rev_parse(zephyr_ws, 'HEAD')
    assert (zephyr_ws / 'should-clone').check(file=1)
