# Copyright 2012 Hewlett-Packard Development Company, L.P.
# Copyright 2021-2022 Acme Gating, LLC
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import configparser
import gc
import json
import logging
import os
import re
import shutil
import socket
import textwrap
import threading
import time
from collections import namedtuple
from unittest import mock, skip
from uuid import uuid4
from kazoo.exceptions import NoNodeError
from testtools.matchers import StartsWith

import git
import fixtures

import zuul.change_matcher
from zuul.driver.gerrit import gerritreporter
import zuul.scheduler
import zuul.model
import zuul.merger.merger
from zuul.lib import yamlutil as yaml

from tests.base import (
    FIXTURE_DIR,
    RecordingExecutorServer,
    SSLZuulTestCase,
    TestConnectionRegistry,
    ZuulTestCase,
    iterate_timeout,
    okay_tracebacks,
    repack_repo,
    simple_layout,
    skipIfMultiScheduler,
)
from zuul.zk.change_cache import ChangeKey
from zuul.zk.event_queues import PIPELINE_NAME_ROOT
from zuul.zk.layout import LayoutState
from zuul.zk.locks import management_queue_lock, pipeline_lock
from zuul.zk import zkobject

EMPTY_LAYOUT_STATE = LayoutState("", "", 0, None, {}, -1)


class TestSchedulerSSL(SSLZuulTestCase):
    tenant_config_file = 'config/single-tenant/main.yaml'

    def test_jobs_executed(self):
        "Test that jobs are executed and a change is merged"
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        A.addApproval('Code-Review', 2)
        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.waitUntilSettled()
        self.assertEqual(self.getJobFromHistory('project-merge').result,
                         'SUCCESS')
        self.assertEqual(self.getJobFromHistory('project-test1').result,
                         'SUCCESS')
        self.assertEqual(self.getJobFromHistory('project-test2').result,
                         'SUCCESS')
        self.assertEqual(A.data['status'], 'MERGED')
        self.assertEqual(A.reported, 2)
        self.assertEqual(self.getJobFromHistory('project-test1').node,
                         'label1')
        self.assertEqual(self.getJobFromHistory('project-test2').node,
                         'label1')


class TestSchedulerZone(ZuulTestCase):
    tenant_config_file = 'config/single-tenant/main.yaml'

    def setUp(self):
        super(TestSchedulerZone, self).setUp()
        self.fake_nodepool.attributes = {'executor-zone': 'test-provider.vpn'}

        # Create an unzoned executor
        config = configparser.ConfigParser()
        config.read_dict(self.config)
        config.remove_option('executor', 'zone')
        config.set('executor', 'command_socket',
                   os.path.join(self.test_root, 'executor2.socket'))
        executor_connections = TestConnectionRegistry(
            self.config, self.test_config, self.additional_event_queues,
            self.upstream_root, self.poller_events,
            self.git_url_with_auth, self.addCleanup)
        executor_connections.configure(self.config, sources=True)
        self.executor_server_unzoned = RecordingExecutorServer(
            config,
            connections=executor_connections,
            jobdir_root=self.jobdir_root,
            _run_ansible=self.run_ansible,
            _test_root=self.test_root,
            log_console_port=self.log_console_port)
        self.executor_server_unzoned.start()
        self.addCleanup(self._shutdown_executor)

    def _shutdown_executor(self):
        self.executor_server_unzoned.hold_jobs_in_build = False
        self.executor_server_unzoned.release()
        self.executor_server_unzoned.stop()
        self.executor_server_unzoned.join()

    def setup_config(self, config_file: str):
        config = super(TestSchedulerZone, self).setup_config(config_file)
        config.set('executor', 'zone', 'test-provider.vpn')
        return config

    def test_jobs_executed(self):
        "Test that jobs are executed and a change is merged per zone"

        # Validate that the reported executor stats are correct. There must
        # be two executors online (one unzoned and one zoned)
        # TODO(corvus): remove deprecated top-level stats in 5.0
        self.assertReportedStat(
            'zuul.executors.online', value='2', kind='g')
        self.assertReportedStat(
            'zuul.executors.unzoned.online', value='1', kind='g')
        self.assertReportedStat(
            'zuul.executors.zone.test-provider_vpn.online',
            value='1', kind='g')

        self.hold_jobs_in_queue = True
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        A.addApproval('Code-Review', 2)
        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.waitUntilSettled()

        queue = list(self.executor_api.queued())
        self.assertEqual(len(self.builds), 0)
        self.assertEqual(len(queue), 1)
        self.assertEqual('test-provider.vpn', queue[0].zone)

        self.hold_jobs_in_queue = False
        self.executor_api.release()
        self.waitUntilSettled()

        self.assertEqual(self.getJobFromHistory('project-merge').result,
                         'SUCCESS')
        self.assertEqual(self.getJobFromHistory('project-test1').result,
                         'SUCCESS')
        self.assertEqual(self.getJobFromHistory('project-test2').result,
                         'SUCCESS')
        self.assertEqual(A.data['status'], 'MERGED')
        self.assertEqual(A.reported, 2)
        self.assertEqual(self.getJobFromHistory('project-test1').node,
                         'label1')
        self.assertEqual(self.getJobFromHistory('project-test2').node,
                         'label1')

        # Validate that both (zoned and unzoned) executors are accepting work
        self.assertReportedStat(
            'zuul.executors.accepting', value='2', kind='g')
        self.assertReportedStat(
            'zuul.executors.unzoned.accepting', value='1', kind='g')
        self.assertReportedStat(
            'zuul.executors.zone.test-provider_vpn.accepting',
            value='1', kind='g')

    def test_executor_disconnect(self):
        "Test that jobs are completed after an executor disconnect"

        self.executor_server.hold_jobs_in_build = True
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        A.addApproval('Code-Review', 2)
        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.waitUntilSettled()

        # Forcibly disconnect the executor from ZK
        self.executor_server.zk_client.client.stop()
        self.executor_server.zk_client.client.start()

        # Find the build in the scheduler so we can check its status
        items = self.getAllItems('tenant-one', 'gate')
        builds = items[0].current_build_set.getBuilds()
        build = builds[0]

        # Clean up the build
        self.scheds.first.sched.executor.cleanupLostBuildRequests()
        # Wait for the build to be reported as lost
        for x in iterate_timeout(30, 'retry build'):
            if build.result == 'RETRY':
                break

        # If we didn't timeout, then it worked; we're done
        self.executor_server.hold_jobs_in_build = False
        self.executor_server.release()
        self.waitUntilSettled()

        # There is a test-only race in the recording executor class
        # where we may record a successful first build, even though
        # the executor didn't actually send a build complete event.
        # This could probabyl be improved, but for now, it's
        # sufficient to verify that the job was retried.  So we omit a
        # result classifier on the first build.
        self.assertHistory([
            dict(name='project-merge', changes='1,1'),
            dict(name='project-merge', result='SUCCESS', changes='1,1'),
            dict(name='project-test1', result='SUCCESS', changes='1,1'),
            dict(name='project-test2', result='SUCCESS', changes='1,1'),
        ], ordered=False)


class TestSchedulerZoneFallback(ZuulTestCase):
    tenant_config_file = 'config/single-tenant/main.yaml'

    def setup_config(self, config_file: str):
        config = super().setup_config(config_file)
        config.set('executor', 'zone', 'test-provider.vpn')
        config.set('executor', 'allow_unzoned', 'true')
        return config

    def test_jobs_executed(self):
        "Test that jobs are executed and a change is merged per zone"
        self.hold_jobs_in_queue = True

        # Validate that the reported executor stats are correct. Since
        # the executor accepts zoned and unzoned job it should be counted
        # in both metrics.
        self.assertReportedStat(
            'zuul.executors.online', value='1', kind='g')
        self.assertReportedStat(
            'zuul.executors.unzoned.online', value='1', kind='g')
        self.assertReportedStat(
            'zuul.executors.zone.test-provider_vpn.online',
            value='1', kind='g')

        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        A.addApproval('Code-Review', 2)
        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.waitUntilSettled()

        queue = list(self.executor_api.queued())
        self.assertEqual(len(self.builds), 0)
        self.assertEqual(len(queue), 1)
        self.assertEqual(None, queue[0].zone)

        self.hold_jobs_in_queue = False
        self.executor_api.release()
        self.waitUntilSettled()

        self.assertEqual(self.getJobFromHistory('project-merge').result,
                         'SUCCESS')
        self.assertEqual(self.getJobFromHistory('project-test1').result,
                         'SUCCESS')
        self.assertEqual(self.getJobFromHistory('project-test2').result,
                         'SUCCESS')
        self.assertEqual(A.data['status'], 'MERGED')
        self.assertEqual(A.reported, 2)
        self.assertEqual(self.getJobFromHistory('project-test1').node,
                         'label1')
        self.assertEqual(self.getJobFromHistory('project-test2').node,
                         'label1')


class TestSchedulerAutoholdHoldExpiration(ZuulTestCase):
    '''
    This class of tests validates the autohold node expiration values
    are set correctly via zuul config or from a custom value.
    '''
    config_file = 'zuul-hold-expiration.conf'
    tenant_config_file = 'config/single-tenant/main.yaml'

    @simple_layout('layouts/autohold.yaml')
    def test_autohold_max_hold_default(self):
        '''
        Test that the hold request node expiration will default to the
        value specified in the configuration file.
        '''
        # Add a autohold with no hold expiration.
        self.scheds.first.sched.autohold(
            'tenant-one', 'review.example.com/org/project', 'project-test2',
            ".*", "reason text", 1, None)

        # There should be a record in ZooKeeper
        request_list = self.sched_zk_nodepool.getHoldRequests()
        self.assertEqual(1, len(request_list))
        request = self.sched_zk_nodepool.getHoldRequest(
            request_list[0])
        self.assertIsNotNone(request)
        self.assertEqual('tenant-one', request.tenant)
        self.assertEqual('review.example.com/org/project', request.project)
        self.assertEqual('project-test2', request.job)
        self.assertEqual('reason text', request.reason)
        self.assertEqual(1, request.max_count)
        self.assertEqual(0, request.current_count)
        self.assertEqual([], request.nodes)
        # This should be the default value from the zuul config file.
        self.assertEqual(1800, request.node_expiration)

    @simple_layout('layouts/autohold.yaml')
    def test_autohold_max_hold_custom(self):
        '''
        Test that the hold request node expiration will be set to the custom
        value specified in the request.
        '''
        # Add a autohold with a custom hold expiration.
        self.scheds.first.sched.autohold(
            'tenant-one', 'review.example.com/org/project', 'project-test2',
            ".*", "reason text", 1, 500)

        # There should be a record in ZooKeeper
        request_list = self.sched_zk_nodepool.getHoldRequests()
        self.assertEqual(1, len(request_list))
        request = self.sched_zk_nodepool.getHoldRequest(
            request_list[0])
        self.assertIsNotNone(request)
        self.assertEqual('tenant-one', request.tenant)
        self.assertEqual('review.example.com/org/project', request.project)
        self.assertEqual('project-test2', request.job)
        self.assertEqual('reason text', request.reason)
        self.assertEqual(1, request.max_count)
        self.assertEqual(0, request.current_count)
        self.assertEqual([], request.nodes)
        # This should be the value from the user request.
        self.assertEqual(500, request.node_expiration)

    @simple_layout('layouts/autohold.yaml')
    def test_autohold_max_hold_custom_invalid(self):
        '''
        Test that if the custom hold request node expiration is higher than our
        configured max, it will be lowered to the max.
        '''
        # Add a autohold with a custom hold expiration that is higher than our
        # configured max.
        self.scheds.first.sched.autohold(
            'tenant-one', 'review.example.com/org/project', 'project-test2',
            ".*", "reason text", 1, 10000)

        # There should be a record in ZooKeeper
        request_list = self.sched_zk_nodepool.getHoldRequests()
        self.assertEqual(1, len(request_list))
        request = self.sched_zk_nodepool.getHoldRequest(
            request_list[0])
        self.assertIsNotNone(request)
        self.assertEqual('tenant-one', request.tenant)
        self.assertEqual('review.example.com/org/project', request.project)
        self.assertEqual('project-test2', request.job)
        self.assertEqual('reason text', request.reason)
        self.assertEqual(1, request.max_count)
        self.assertEqual(0, request.current_count)
        self.assertEqual([], request.nodes)
        # This should be the max value from the zuul config file.
        self.assertEqual(3600, request.node_expiration)


class TestScheduler(ZuulTestCase):
    tenant_config_file = 'config/single-tenant/main.yaml'

    def test_jobs_executed(self):
        "Test that jobs are executed and a change is merged"

        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        A.addApproval('Code-Review', 2)
        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.waitUntilSettled()
        self.assertEqual(self.getJobFromHistory('project-merge').result,
                         'SUCCESS')
        self.assertEqual(self.getJobFromHistory('project-test1').result,
                         'SUCCESS')
        self.assertEqual(self.getJobFromHistory('project-test2').result,
                         'SUCCESS')
        self.assertEqual(A.data['status'], 'MERGED')
        self.assertEqual(A.reported, 2)
        self.assertEqual(self.getJobFromHistory('project-test1').node,
                         'label1')
        self.assertEqual(self.getJobFromHistory('project-test2').node,
                         'label1')
        self.assertThat(A.messages[1],
                        StartsWith(
                        'Build succeeded (gate).\n'
                        'https://zuul.example.com/t/tenant-one/buildset'))

        # TODOv3(jeblair): we may want to report stats by tenant (also?).
        # Per-driver
        self.assertReportedStat('zuul.event.gerrit.comment-added', value='1',
                                kind='c')
        # Per-driver per-connection
        self.assertReportedStat('zuul.event.gerrit.gerrit.comment-added',
                                value='1', kind='c')
        self.assertReportedStat(
            'zuul.tenant.tenant-one.trigger_events', value='0', kind='g')
        self.assertReportedStat(
            'zuul.tenant.tenant-one.management_events', value='0', kind='g')
        self.assertReportedStat(
            'zuul.tenant.tenant-one.pipeline.gate.current_changes',
            value='1', kind='g')
        self.assertReportedStat(
            'zuul.tenant.tenant-one.pipeline.gate.window',
            value='20', kind='g')
        self.assertReportedStat(
            'zuul.tenant.tenant-one.pipeline.gate.queue.'
            'org.project.current_changes',
            value='1', kind='g')
        self.assertReportedStat(
            'zuul.tenant.tenant-one.pipeline.gate.queue.org.project.window',
            value='20', kind='g')
        self.assertReportedStat(
            'zuul.tenant.tenant-one.pipeline.gate.queue.org.project.window',
            value='21', kind='g')
        self.assertReportedStat(
            'zuul.tenant.tenant-one.pipeline.gate.project.review_example_com.'
            'org_project.master.job.project-merge.SUCCESS', kind='ms')
        self.assertReportedStat(
            'zuul.tenant.tenant-one.pipeline.gate.project.review_example_com.'
            'org_project.master.job.project-merge.SUCCESS', value='1',
            kind='c')
        self.assertReportedStat(
            'zuul.tenant.tenant-one.pipeline.gate.resident_time', kind='ms')
        self.assertReportedStat(
            'zuul.tenant.tenant-one.pipeline.gate.total_changes', value='1',
            kind='c')
        self.assertReportedStat(
            'zuul.tenant.tenant-one.pipeline.gate.queue.'
            'org.project.resident_time', kind='ms')
        self.assertReportedStat(
            'zuul.tenant.tenant-one.pipeline.gate.queue.'
            'org.project.total_changes', value='1',
            kind='c')
        self.assertReportedStat(
            'zuul.tenant.tenant-one.pipeline.gate.trigger_events',
            value='0', kind='g')
        self.assertReportedStat(
            'zuul.tenant.tenant-one.pipeline.gate.result_events',
            value='0', kind='g')
        self.assertReportedStat(
            'zuul.tenant.tenant-one.pipeline.gate.management_events',
            value='0', kind='g')
        self.assertReportedStat(
            'zuul.tenant.tenant-one.pipeline.gate.project.review_example_com.'
            'org_project.master.resident_time', kind='ms')
        self.assertReportedStat(
            'zuul.tenant.tenant-one.pipeline.gate.project.review_example_com.'
            'org_project.master.total_changes', value='1', kind='c')
        exec_key = 'zuul.executor.%s' % self.executor_server.hostname.replace(
            '.', '_')
        self.assertReportedStat(exec_key + '.builds', value='1', kind='c')
        self.assertReportedStat(exec_key + '.starting_builds', kind='g')
        self.assertReportedStat(exec_key + '.starting_builds', kind='ms')
        self.assertReportedStat(
            'zuul.nodepool.requests.requested.total', value='1', kind='c')
        self.assertReportedStat(
            'zuul.nodepool.requests.requested.label.label1',
            value='1', kind='c')
        self.assertReportedStat(
            'zuul.nodepool.requests.fulfilled.label.label1',
            value='1', kind='c')
        self.assertReportedStat(
            'zuul.nodepool.requests.requested.size.1', value='1', kind='c')
        self.assertReportedStat(
            'zuul.nodepool.requests.fulfilled.size.1', value='1', kind='c')
        # just check for existence, since we can not know if a request is
        # in-flight during the sched._stats_inverval
        self.assertReportedStat(
            'zuul.nodepool.current_requests', kind='g')
        self.assertReportedStat(
            'zuul.nodepool.tenant.tenant-one.current_requests', kind='g')
        self.assertReportedStat(
            'zuul.executors.online', value='1', kind='g')
        self.assertReportedStat(
            'zuul.executors.accepting', value='1', kind='g')
        self.assertReportedStat(
            'zuul.mergers.online', value='1', kind='g')
        self.assertReportedStat('zuul.scheduler.eventqueues.connection.gerrit',
                                value='0', kind='g')
        self.assertReportedStat('zuul.scheduler.run_handler', kind='ms')

        # Catch time / monotonic errors
        for key in [
                'zuul.tenant.tenant-one.event_enqueue_processing_time',
                'zuul.tenant.tenant-one.event_enqueue_time',
                'zuul.tenant.tenant-one.reconfiguration_time',
                'zuul.tenant.tenant-one.pipeline.gate.event_enqueue_time',
                'zuul.tenant.tenant-one.pipeline.gate.merge_request_time',
                'zuul.tenant.tenant-one.pipeline.gate.merger_merge_op_time',
                'zuul.tenant.tenant-one.pipeline.gate.job_freeze_time',
                'zuul.tenant.tenant-one.pipeline.gate.node_request_time',
                'zuul.tenant.tenant-one.pipeline.gate.job_wait_time',
                'zuul.tenant.tenant-one.pipeline.gate.event_job_time',
                'zuul.tenant.tenant-one.pipeline.gate.resident_time',
                'zuul.tenant.tenant-one.pipeline.gate.read_time',
                'zuul.tenant.tenant-one.pipeline.gate.write_time',
                'zuul.tenant.tenant-one.pipeline.gate.process',
                'zuul.tenant.tenant-one.pipeline.gate.event_process',
                'zuul.tenant.tenant-one.pipeline.gate.handling',
                'zuul.tenant.tenant-one.pipeline.gate.refresh',
        ]:
            val = self.assertReportedStat(key, kind='ms')
            self.assertTrue(0.0 < float(val) < 60000.0)

        for key in [
                'zuul.tenant.tenant-one.pipeline.gate.read_objects',
                'zuul.tenant.tenant-one.pipeline.gate.write_objects',
                'zuul.tenant.tenant-one.pipeline.gate.read_znodes',
                'zuul.tenant.tenant-one.pipeline.gate.write_znodes',
                'zuul.tenant.tenant-one.pipeline.gate.write_bytes',
        ]:
            # 'zuul.tenant.tenant-one.pipeline.gate.read_bytes' is
            # expected to be zero since it's initialized after reading
            val = self.assertReportedStat(key, kind='g')
            self.assertTrue(0.0 < float(val) < 60000.0)

        self.assertReportedStat('zuul.tenant.tenant-one.pipeline.gate.'
                                'data_size_compressed',
                                kind='g')
        self.assertReportedStat('zuul.tenant.tenant-one.pipeline.gate.'
                                'data_size_uncompressed',
                                kind='g')
        self.assertReportedStat('zuul.connection.gerrit.cache.'
                                'data_size_compressed',
                                kind='g')
        self.assertReportedStat('zuul.connection.gerrit.cache.'
                                'data_size_uncompressed',
                                kind='g')

        for build in self.history:
            self.assertTrue(build.parameters['zuul']['voting'])

    def test_zk_profile(self):
        command_socket = self.scheds.first.sched.config.get(
            'scheduler', 'command_socket')
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        zplog = logging.getLogger('zuul.profile')
        with self.assertLogs('zuul.profile', level='DEBUG') as logs:
            self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
            self.waitUntilSettled()
            # assertNoLogs doesn't appear until py3.10, so we need to
            # emit a single log line in order to assert that there
            # aren't any others.
            zplog.debug('test')
            self.assertEqual(1, len(logs.output))
        args = json.dumps(['tenant-one', 'check'])
        with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as s:
            s.connect(command_socket)
            s.sendall(f'zkprofile {args}\n'.encode('utf8'))
        with self.assertLogs('zuul.profile', level='DEBUG'):
            self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
            self.waitUntilSettled()
        with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as s:
            s.connect(command_socket)
            s.sendall(f'zkprofile {args}\n'.encode('utf8'))
        with self.assertLogs('zuul.profile', level='DEBUG') as logs:
            self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
            self.waitUntilSettled()
            zplog.debug('test')
            self.assertEqual(1, len(logs.output))

    def test_initial_pipeline_gauges(self):
        "Test that each pipeline reported its length on start"
        self.assertReportedStat('zuul.tenant.tenant-one.pipeline.gate.'
                                'current_changes',
                                value='0', kind='g')
        self.assertReportedStat('zuul.tenant.tenant-one.pipeline.check.'
                                'current_changes',
                                value='0', kind='g')

    def test_job_branch(self):
        "Test the correct variant of a job runs on a branch"
        self.create_branch('org/project', 'stable')
        self.fake_gerrit.addEvent(
            self.fake_gerrit.getFakeBranchCreatedEvent(
                'org/project', 'stable'))
        self.waitUntilSettled()

        A = self.fake_gerrit.addFakeChange('org/project', 'stable', 'A')
        A.addApproval('Code-Review', 2)
        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.waitUntilSettled()
        self.assertEqual(self.getJobFromHistory('project-test1').result,
                         'SUCCESS')
        self.assertEqual(self.getJobFromHistory('project-test2').result,
                         'SUCCESS')
        self.assertEqual(A.data['status'], 'MERGED')
        self.assertEqual(A.reported, 2,
                         "A should report start and success")
        self.assertIn('gate', A.messages[1],
                      "A should transit gate")
        self.assertEqual(self.getJobFromHistory('project-test1').node,
                         'label2')

    @simple_layout('layouts/branch-deletion.yaml')
    def test_branch_deletion(self):
        "Test the correct variant of a job runs on a branch"

        # Start a secondary merger so this test exercises branch
        # deletion on both a merger and a separate executor.
        self.executor_server._merger_running = False
        self.executor_server.merger_loop_wake_event.set()
        self.executor_server.merger_thread.join()
        self._startMerger()

        self.create_branch('org/project', 'stable')
        self.fake_gerrit.addEvent(
            self.fake_gerrit.getFakeBranchCreatedEvent(
                'org/project', 'stable'))
        self.waitUntilSettled()
        A = self.fake_gerrit.addFakeChange('org/project', 'stable', 'A')
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()
        self.assertEqual(self.getJobFromHistory('project-test2').result,
                         'SUCCESS')

        self.delete_branch('org/project', 'stable')
        path = os.path.join(self.executor_src_root, 'review.example.com')
        shutil.rmtree(path)

        self.executor_server.hold_jobs_in_build = True
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()
        build = self.builds[0]

        # Make sure there is no stable branch in the checked out git repo.
        pname = 'review.example.com/org/project'
        work = build.getWorkspaceRepos([pname])
        work = work[pname]
        heads = set([str(x) for x in work.heads])
        self.assertEqual(heads, set(['master']))
        self.executor_server.hold_jobs_in_build = False
        build.release()
        self.waitUntilSettled()
        self.assertEqual(self.getJobFromHistory('project-test1').result,
                         'SUCCESS')

    @simple_layout('layouts/branch-repo-state.yaml')
    def test_branch_repo_state(self):
        """
        Test that we schedule no initial merge for branch/ref events
        and the global repo state only if there are jobs to run.
        """
        del self.merge_job_history

        A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
        A.setMerged()
        self.fake_gerrit.addEvent(A.getRefUpdatedEvent())
        self.waitUntilSettled()

        # No merge expected as there are not jobs for project1 in the
        # post pipeline.
        self.assertIsNone(self.merge_job_history.get(
            zuul.model.MergeRequest.REF_STATE))

        B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
        B.setMerged()
        self.fake_gerrit.addEvent(B.getRefUpdatedEvent())
        self.waitUntilSettled()

        # One refstate merge request for the global repo state since
        # there are jobs configured for project2 in the post pipeline.
        refstate_jobs = self.merge_job_history.get(
            zuul.model.MergeRequest.REF_STATE)
        self.assertEqual(len(refstate_jobs), 1)

        merge_request = refstate_jobs[0]
        merge_items = {i["project"] for i in merge_request.payload["items"]}
        self.assertEqual(
            merge_items,
            {"org/project1", "org/project2", "org/common-config"})

    def test_parallel_changes(self):
        "Test that changes are tested in parallel and merged in series"

        self.executor_server.hold_jobs_in_build = True
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
        C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
        A.addApproval('Code-Review', 2)
        B.addApproval('Code-Review', 2)
        C.addApproval('Code-Review', 2)

        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
        self.fake_gerrit.addEvent(C.addApproval('Approved', 1))

        self.waitUntilSettled()
        self.assertEqual(len(self.builds), 1)
        self.assertEqual(self.builds[0].name, 'project-merge')
        self.assertTrue(self.builds[0].hasChanges(A))

        self.executor_server.release('.*-merge')
        self.waitUntilSettled()
        self.assertEqual(len(self.builds), 3)
        self.assertEqual(self.builds[0].name, 'project-test1')
        self.assertTrue(self.builds[0].hasChanges(A))
        self.assertEqual(self.builds[1].name, 'project-test2')
        self.assertTrue(self.builds[1].hasChanges(A))
        self.assertEqual(self.builds[2].name, 'project-merge')
        self.assertTrue(self.builds[2].hasChanges(A, B))

        self.executor_server.release('.*-merge')
        self.waitUntilSettled()
        self.assertEqual(len(self.builds), 5)
        self.assertEqual(self.builds[0].name, 'project-test1')
        self.assertTrue(self.builds[0].hasChanges(A))
        self.assertEqual(self.builds[1].name, 'project-test2')
        self.assertTrue(self.builds[1].hasChanges(A))

        self.assertEqual(self.builds[2].name, 'project-test1')
        self.assertTrue(self.builds[2].hasChanges(A, B))
        self.assertEqual(self.builds[3].name, 'project-test2')
        self.assertTrue(self.builds[3].hasChanges(A, B))

        self.assertEqual(self.builds[4].name, 'project-merge')
        self.assertTrue(self.builds[4].hasChanges(A, B, C))

        self.executor_server.release('.*-merge')
        self.waitUntilSettled()
        self.assertEqual(len(self.builds), 6)
        self.assertEqual(self.builds[0].name, 'project-test1')
        self.assertTrue(self.builds[0].hasChanges(A))
        self.assertEqual(self.builds[1].name, 'project-test2')
        self.assertTrue(self.builds[1].hasChanges(A))

        self.assertEqual(self.builds[2].name, 'project-test1')
        self.assertTrue(self.builds[2].hasChanges(A, B))
        self.assertEqual(self.builds[3].name, 'project-test2')
        self.assertTrue(self.builds[3].hasChanges(A, B))

        self.assertEqual(self.builds[4].name, 'project-test1')
        self.assertTrue(self.builds[4].hasChanges(A, B, C))
        self.assertEqual(self.builds[5].name, 'project-test2')
        self.assertTrue(self.builds[5].hasChanges(A, B, C))

        self.executor_server.hold_jobs_in_build = False
        self.executor_server.release()
        self.waitUntilSettled()
        self.assertEqual(len(self.builds), 0)

        self.assertEqual(len(self.history), 9)
        self.assertEqual(A.data['status'], 'MERGED')
        self.assertEqual(B.data['status'], 'MERGED')
        self.assertEqual(C.data['status'], 'MERGED')
        self.assertEqual(A.reported, 2)
        self.assertEqual(B.reported, 2)
        self.assertEqual(C.reported, 2)

    def test_failed_changes(self):
        "Test that a change behind a failed change is retested"
        self.executor_server.hold_jobs_in_build = True

        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
        A.addApproval('Code-Review', 2)
        B.addApproval('Code-Review', 2)

        self.executor_server.failJob('project-test1', A)

        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
        self.waitUntilSettled()
        self.assertBuilds([dict(name='project-merge', changes='1,1')])

        self.executor_server.release('.*-merge')
        self.waitUntilSettled()
        # A/project-merge is complete
        self.assertBuilds([
            dict(name='project-test1', changes='1,1'),
            dict(name='project-test2', changes='1,1'),
            dict(name='project-merge', changes='1,1 2,1'),
        ])

        self.executor_server.release('.*-merge')
        self.waitUntilSettled()
        # A/project-merge is complete
        # B/project-merge is complete
        self.assertBuilds([
            dict(name='project-test1', changes='1,1'),
            dict(name='project-test2', changes='1,1'),
            dict(name='project-test1', changes='1,1 2,1'),
            dict(name='project-test2', changes='1,1 2,1'),
        ])

        # Release project-test1 for A which will fail.  This will
        # abort both running B jobs and reexecute project-merge for B.
        self.builds[0].release()
        self.waitUntilSettled()

        self.orderedRelease()
        self.assertHistory([
            dict(name='project-merge', result='SUCCESS', changes='1,1'),
            dict(name='project-merge', result='SUCCESS', changes='1,1 2,1'),
            dict(name='project-test1', result='FAILURE', changes='1,1'),
            dict(name='project-test1', result='ABORTED', changes='1,1 2,1'),
            dict(name='project-test2', result='ABORTED', changes='1,1 2,1'),
            dict(name='project-test2', result='SUCCESS', changes='1,1'),
            dict(name='project-merge', result='SUCCESS', changes='2,1'),
            dict(name='project-test1', result='SUCCESS', changes='2,1'),
            dict(name='project-test2', result='SUCCESS', changes='2,1'),
        ], ordered=False)

        self.assertEqual(A.data['status'], 'NEW')
        self.assertEqual(B.data['status'], 'MERGED')
        self.assertEqual(A.reported, 2)
        self.assertEqual(B.reported, 2)

        # Make sure that after a reset of the buildset the files state is
        # updated correctly and the schedulers is not resolving the list
        # of changed files via the merger.
        self.assertIsNone(
            self.scheds.first.sched.merger.merger_api.history.get(
                "fileschanges"))

    def test_independent_queues(self):
        "Test that changes end up in the right queues"

        self.executor_server.hold_jobs_in_build = True
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
        C = self.fake_gerrit.addFakeChange('org/project2', 'master', 'C')
        A.addApproval('Code-Review', 2)
        B.addApproval('Code-Review', 2)
        C.addApproval('Code-Review', 2)

        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.waitUntilSettled()

        self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
        self.waitUntilSettled()

        self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
        self.waitUntilSettled()

        # There should be one merge job at the head of each queue running
        self.assertBuilds([
            dict(name='project-merge', changes='1,1'),
            dict(name='project-merge', changes='2,1'),
        ])

        # Release the current merge builds
        self.builds[0].release()
        self.waitUntilSettled()
        self.builds[0].release()
        self.waitUntilSettled()
        # Release the merge job for project2 which is behind project1
        self.executor_server.release('.*-merge')
        self.waitUntilSettled()

        # All the test builds should be running:
        self.assertBuilds([
            dict(name='project-test1', changes='1,1'),
            dict(name='project-test2', changes='1,1'),
            dict(name='project-test1', changes='2,1'),
            dict(name='project-test2', changes='2,1'),
            dict(name='project1-project2-integration', changes='2,1'),
            dict(name='project-test1', changes='2,1 3,1'),
            dict(name='project-test2', changes='2,1 3,1'),
            dict(name='project1-project2-integration', changes='2,1 3,1'),
        ])

        self.orderedRelease()
        self.assertHistory([
            dict(name='project-merge', result='SUCCESS', changes='1,1'),
            dict(name='project-merge', result='SUCCESS', changes='2,1'),
            dict(name='project-merge', result='SUCCESS', changes='2,1 3,1'),
            dict(name='project-test1', result='SUCCESS', changes='1,1'),
            dict(name='project-test2', result='SUCCESS', changes='1,1'),
            dict(name='project-test1', result='SUCCESS', changes='2,1'),
            dict(name='project-test2', result='SUCCESS', changes='2,1'),
            dict(
                name='project1-project2-integration',
                result='SUCCESS',
                changes='2,1'),
            dict(name='project-test1', result='SUCCESS', changes='2,1 3,1'),
            dict(name='project-test2', result='SUCCESS', changes='2,1 3,1'),
            dict(name='project1-project2-integration',
                 result='SUCCESS',
                 changes='2,1 3,1'),
        ])

        self.assertEqual(A.data['status'], 'MERGED')
        self.assertEqual(B.data['status'], 'MERGED')
        self.assertEqual(C.data['status'], 'MERGED')
        self.assertEqual(A.reported, 2)
        self.assertEqual(B.reported, 2)
        self.assertEqual(C.reported, 2)

    def test_failed_change_at_head(self):
        "Test that if a change at the head fails, jobs behind it are canceled"

        self.executor_server.hold_jobs_in_build = True
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
        C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
        A.addApproval('Code-Review', 2)
        B.addApproval('Code-Review', 2)
        C.addApproval('Code-Review', 2)

        self.executor_server.failJob('project-test1', A)

        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
        self.fake_gerrit.addEvent(C.addApproval('Approved', 1))

        self.waitUntilSettled()

        self.assertBuilds([
            dict(name='project-merge', changes='1,1'),
        ])

        self.executor_server.release('.*-merge')
        self.waitUntilSettled()
        self.executor_server.release('.*-merge')
        self.waitUntilSettled()
        self.executor_server.release('.*-merge')
        self.waitUntilSettled()

        self.assertBuilds([
            dict(name='project-test1', changes='1,1'),
            dict(name='project-test2', changes='1,1'),
            dict(name='project-test1', changes='1,1 2,1'),
            dict(name='project-test2', changes='1,1 2,1'),
            dict(name='project-test1', changes='1,1 2,1 3,1'),
            dict(name='project-test2', changes='1,1 2,1 3,1'),
        ])

        self.release(self.builds[0])
        self.waitUntilSettled()

        # project-test2, project-merge for B
        self.assertBuilds([
            dict(name='project-test2', changes='1,1'),
            dict(name='project-merge', changes='2,1'),
        ])
        # Unordered history comparison because the aborts can finish
        # in any order.
        self.assertHistory([
            dict(name='project-merge', result='SUCCESS',
                 changes='1,1'),
            dict(name='project-merge', result='SUCCESS',
                 changes='1,1 2,1'),
            dict(name='project-merge', result='SUCCESS',
                 changes='1,1 2,1 3,1'),
            dict(name='project-test1', result='FAILURE',
                 changes='1,1'),
            dict(name='project-test1', result='ABORTED',
                 changes='1,1 2,1'),
            dict(name='project-test2', result='ABORTED',
                 changes='1,1 2,1'),
            dict(name='project-test1', result='ABORTED',
                 changes='1,1 2,1 3,1'),
            dict(name='project-test2', result='ABORTED',
                 changes='1,1 2,1 3,1'),
        ], ordered=False)

        self.executor_server.release('.*-merge')
        self.waitUntilSettled()
        self.executor_server.release('.*-merge')
        self.waitUntilSettled()
        self.orderedRelease()

        self.assertBuilds([])
        self.assertHistory([
            dict(name='project-merge', result='SUCCESS',
                 changes='1,1'),
            dict(name='project-merge', result='SUCCESS',
                 changes='1,1 2,1'),
            dict(name='project-merge', result='SUCCESS',
                 changes='1,1 2,1 3,1'),
            dict(name='project-test1', result='FAILURE',
                 changes='1,1'),
            dict(name='project-test1', result='ABORTED',
                 changes='1,1 2,1'),
            dict(name='project-test2', result='ABORTED',
                 changes='1,1 2,1'),
            dict(name='project-test1', result='ABORTED',
                 changes='1,1 2,1 3,1'),
            dict(name='project-test2', result='ABORTED',
                 changes='1,1 2,1 3,1'),
            dict(name='project-merge', result='SUCCESS',
                 changes='2,1'),
            dict(name='project-merge', result='SUCCESS',
                 changes='2,1 3,1'),
            dict(name='project-test2', result='SUCCESS',
                 changes='1,1'),
            dict(name='project-test1', result='SUCCESS',
                 changes='2,1'),
            dict(name='project-test2', result='SUCCESS',
                 changes='2,1'),
            dict(name='project-test1', result='SUCCESS',
                 changes='2,1 3,1'),
            dict(name='project-test2', result='SUCCESS',
                 changes='2,1 3,1'),
        ], ordered=False)

        self.assertEqual(A.data['status'], 'NEW')
        self.assertEqual(B.data['status'], 'MERGED')
        self.assertEqual(C.data['status'], 'MERGED')
        self.assertEqual(A.reported, 2)
        self.assertEqual(B.reported, 2)
        self.assertEqual(C.reported, 2)

    def test_failed_change_in_middle(self):
        "Test a failed change in the middle of the queue"

        self.executor_server.hold_jobs_in_build = True
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
        C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
        A.addApproval('Code-Review', 2)
        B.addApproval('Code-Review', 2)
        C.addApproval('Code-Review', 2)

        self.executor_server.failJob('project-test1', B)

        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
        self.fake_gerrit.addEvent(C.addApproval('Approved', 1))

        self.waitUntilSettled()

        self.executor_server.release('.*-merge')
        self.waitUntilSettled()
        self.executor_server.release('.*-merge')
        self.waitUntilSettled()
        self.executor_server.release('.*-merge')
        self.waitUntilSettled()

        self.assertEqual(len(self.builds), 6)
        self.assertEqual(self.builds[0].name, 'project-test1')
        self.assertEqual(self.builds[1].name, 'project-test2')
        self.assertEqual(self.builds[2].name, 'project-test1')
        self.assertEqual(self.builds[3].name, 'project-test2')
        self.assertEqual(self.builds[4].name, 'project-test1')
        self.assertEqual(self.builds[5].name, 'project-test2')

        self.release(self.builds[2])
        self.waitUntilSettled()

        # project-test1 and project-test2 for A
        # project-test2 for B
        # project-merge for C (without B)
        self.assertEqual(len(self.builds), 4)
        self.assertEqual(self.countJobResults(self.history, 'ABORTED'), 2)

        self.executor_server.release('.*-merge')
        self.waitUntilSettled()

        # project-test1 and project-test2 for A
        # project-test2 for B
        # project-test1 and project-test2 for C
        self.assertEqual(len(self.builds), 5)

        items = self.getAllItems('tenant-one', 'gate')
        builds = items[0].current_build_set.getBuilds()
        self.assertEqual(self.countJobResults(builds, 'SUCCESS'), 1)
        self.assertEqual(self.countJobResults(builds, None), 2)
        builds = items[1].current_build_set.getBuilds()
        self.assertEqual(self.countJobResults(builds, 'SUCCESS'), 1)
        self.assertEqual(self.countJobResults(builds, 'FAILURE'), 1)
        self.assertEqual(self.countJobResults(builds, None), 1)
        builds = items[2].current_build_set.getBuilds()
        self.assertEqual(self.countJobResults(builds, 'SUCCESS'), 1)
        self.assertEqual(self.countJobResults(builds, None), 2)

        self.executor_server.hold_jobs_in_build = False
        self.executor_server.release()
        self.waitUntilSettled()

        self.assertEqual(len(self.builds), 0)
        self.assertEqual(len(self.history), 12)
        self.assertEqual(A.data['status'], 'MERGED')
        self.assertEqual(B.data['status'], 'NEW')
        self.assertEqual(C.data['status'], 'MERGED')
        self.assertEqual(A.reported, 2)
        self.assertEqual(B.reported, 2)
        self.assertEqual(C.reported, 2)

    def test_failed_change_at_head_with_queue(self):
        "Test that if a change at the head fails, queued jobs are canceled"

        self.hold_jobs_in_queue = True
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
        C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
        A.addApproval('Code-Review', 2)
        B.addApproval('Code-Review', 2)
        C.addApproval('Code-Review', 2)

        self.executor_server.failJob('project-test1', A)

        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
        self.fake_gerrit.addEvent(C.addApproval('Approved', 1))

        self.waitUntilSettled()
        queue = list(self.executor_api.queued())
        self.assertEqual(len(self.builds), 0)
        self.assertEqual(len(queue), 1)
        self.assertEqual(queue[0].zone, None)
        params = self.executor_server.executor_api.getParams(queue[0])
        self.assertEqual(params['zuul']['job'], 'project-merge')
        self.assertEqual(params['items'][0]['number'], '%d' % A.number)

        self.executor_api.release('.*-merge')
        self.waitUntilSettled()
        self.executor_api.release('.*-merge')
        self.waitUntilSettled()
        self.executor_api.release('.*-merge')
        self.waitUntilSettled()

        queue = list(self.executor_api.queued())
        self.assertEqual(len(self.builds), 0)
        self.assertEqual(len(queue), 6)

        params = [self.executor_server.executor_api.getParams(x)
                  for x in queue]
        self.assertEqual(params[0]['zuul']['job'], 'project-test1')
        self.assertEqual(params[1]['zuul']['job'], 'project-test2')
        self.assertEqual(params[2]['zuul']['job'], 'project-test1')
        self.assertEqual(params[3]['zuul']['job'], 'project-test2')
        self.assertEqual(params[4]['zuul']['job'], 'project-test1')
        self.assertEqual(params[5]['zuul']['job'], 'project-test2')

        self.executor_api.release(queue[0])
        self.waitUntilSettled()

        self.assertEqual(len(self.builds), 0)
        queue = list(self.executor_api.queued())
        self.assertEqual(len(queue), 2)  # project-test2, project-merge for B
        self.assertEqual(self.countJobResults(self.history, 'ABORTED'), 0)

        self.hold_jobs_in_queue = False
        self.executor_api.release()
        self.waitUntilSettled()

        self.assertEqual(len(self.builds), 0)
        self.assertEqual(len(self.history), 11)
        self.assertEqual(A.data['status'], 'NEW')
        self.assertEqual(B.data['status'], 'MERGED')
        self.assertEqual(C.data['status'], 'MERGED')
        self.assertEqual(A.reported, 2)
        self.assertEqual(B.reported, 2)
        self.assertEqual(C.reported, 2)

    def _test_time_database(self, iteration):
        self.executor_server.hold_jobs_in_build = True
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        A.addApproval('Code-Review', 2)
        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.waitUntilSettled()
        time.sleep(2)

        found_job = None
        manager = self.scheds.first.sched.abide.tenants[
            'tenant-one'].layout.pipeline_managers['gate']
        pipeline_status = manager.formatStatusJSON(
            self.scheds.first.sched.globals.websocket_url)
        for queue in pipeline_status['change_queues']:
            for head in queue['heads']:
                for item in head:
                    for job in item['jobs']:
                        if job['name'] == 'project-merge':
                            found_job = job
                            break

        self.assertIsNotNone(found_job)
        if iteration == 1:
            self.assertIsNotNone(found_job['estimated_time'])
            self.assertIsNone(found_job['remaining_time'])
        else:
            self.assertIsNotNone(found_job['estimated_time'])
            self.assertTrue(found_job['estimated_time'] >= 2)
            self.assertIsNotNone(found_job['remaining_time'])

        self.executor_server.hold_jobs_in_build = False
        self.executor_server.release()
        self.waitUntilSettled()

    def test_time_database(self):
        "Test the time database"

        self._test_time_database(1)
        self._test_time_database(2)

    def test_two_failed_changes_at_head(self):
        "Test that changes are reparented correctly if 2 fail at head"

        self.executor_server.hold_jobs_in_build = True
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
        C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
        A.addApproval('Code-Review', 2)
        B.addApproval('Code-Review', 2)
        C.addApproval('Code-Review', 2)

        self.executor_server.failJob('project-test1', A)
        self.executor_server.failJob('project-test1', B)

        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
        self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
        self.waitUntilSettled()

        self.executor_server.release('.*-merge')
        self.waitUntilSettled()
        self.executor_server.release('.*-merge')
        self.waitUntilSettled()
        self.executor_server.release('.*-merge')
        self.waitUntilSettled()

        self.assertEqual(len(self.builds), 6)
        self.assertEqual(self.builds[0].name, 'project-test1')
        self.assertEqual(self.builds[1].name, 'project-test2')
        self.assertEqual(self.builds[2].name, 'project-test1')
        self.assertEqual(self.builds[3].name, 'project-test2')
        self.assertEqual(self.builds[4].name, 'project-test1')
        self.assertEqual(self.builds[5].name, 'project-test2')

        self.assertTrue(self.builds[0].hasChanges(A))
        self.assertTrue(self.builds[2].hasChanges(A))
        self.assertTrue(self.builds[2].hasChanges(B))
        self.assertTrue(self.builds[4].hasChanges(A))
        self.assertTrue(self.builds[4].hasChanges(B))
        self.assertTrue(self.builds[4].hasChanges(C))

        # Fail change B first
        self.release(self.builds[2])
        self.waitUntilSettled()

        # restart of C after B failure
        self.executor_server.release('.*-merge')
        self.waitUntilSettled()

        self.assertEqual(len(self.builds), 5)
        self.assertEqual(self.builds[0].name, 'project-test1')
        self.assertEqual(self.builds[1].name, 'project-test2')
        self.assertEqual(self.builds[2].name, 'project-test2')
        self.assertEqual(self.builds[3].name, 'project-test1')
        self.assertEqual(self.builds[4].name, 'project-test2')

        self.assertTrue(self.builds[1].hasChanges(A))
        self.assertTrue(self.builds[2].hasChanges(A))
        self.assertTrue(self.builds[2].hasChanges(B))
        self.assertTrue(self.builds[4].hasChanges(A))
        self.assertFalse(self.builds[4].hasChanges(B))
        self.assertTrue(self.builds[4].hasChanges(C))

        # Finish running all passing jobs for change A
        self.release(self.builds[1])
        self.waitUntilSettled()
        # Fail and report change A
        self.release(self.builds[0])
        self.waitUntilSettled()

        # restart of B,C after A failure
        self.executor_server.release('.*-merge')
        self.waitUntilSettled()
        self.executor_server.release('.*-merge')
        self.waitUntilSettled()

        self.assertEqual(len(self.builds), 4)
        self.assertEqual(self.builds[0].name, 'project-test1')  # B
        self.assertEqual(self.builds[1].name, 'project-test2')  # B
        self.assertEqual(self.builds[2].name, 'project-test1')  # C
        self.assertEqual(self.builds[3].name, 'project-test2')  # C

        self.assertFalse(self.builds[1].hasChanges(A))
        self.assertTrue(self.builds[1].hasChanges(B))
        self.assertFalse(self.builds[1].hasChanges(C))

        self.assertFalse(self.builds[2].hasChanges(A))
        # After A failed and B and C restarted, B should be back in
        # C's tests because it has not failed yet.
        self.assertTrue(self.builds[2].hasChanges(B))
        self.assertTrue(self.builds[2].hasChanges(C))

        self.executor_server.hold_jobs_in_build = False
        self.executor_server.release()
        self.waitUntilSettled()

        self.assertEqual(len(self.builds), 0)
        self.assertEqual(len(self.history), 21)
        self.assertEqual(A.data['status'], 'NEW')
        self.assertEqual(B.data['status'], 'NEW')
        self.assertEqual(C.data['status'], 'MERGED')
        self.assertEqual(A.reported, 2)
        self.assertEqual(B.reported, 2)
        self.assertEqual(C.reported, 2)

    def test_patch_order(self):
        "Test that dependent patches are tested in the right order"
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
        C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
        A.addApproval('Code-Review', 2)
        B.addApproval('Code-Review', 2)
        C.addApproval('Code-Review', 2)

        M2 = self.fake_gerrit.addFakeChange('org/project', 'master', 'M2')
        M1 = self.fake_gerrit.addFakeChange('org/project', 'master', 'M1')
        M2.setMerged()
        M1.setMerged()

        # C -> B -> A -> M1 -> M2
        # M2 is here to make sure it is never queried.  If it is, it
        # means zuul is walking down the entire history of merged
        # changes.

        C.setDependsOn(B, 1)
        B.setDependsOn(A, 1)
        A.setDependsOn(M1, 1)
        M1.setDependsOn(M2, 1)

        self.fake_gerrit.addEvent(C.addApproval('Approved', 1))

        self.waitUntilSettled()

        self.assertEqual(A.data['status'], 'NEW')
        self.assertEqual(B.data['status'], 'NEW')
        self.assertEqual(C.data['status'], 'NEW')

        self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

        self.waitUntilSettled()
        self.assertEqual(M2.queried, 0)
        self.assertEqual(A.data['status'], 'MERGED')
        self.assertEqual(B.data['status'], 'MERGED')
        self.assertEqual(C.data['status'], 'MERGED')
        self.assertEqual(A.reported, 2)
        self.assertEqual(B.reported, 2)
        self.assertEqual(C.reported, 2)

    def test_needed_changes_enqueue(self):
        "Test that a needed change is enqueued ahead"
        #          A      Given a git tree like this, if we enqueue
        #         / \     change C, we should walk up and down the tree
        #        B   G    and enqueue changes in the order ABCDEFG.
        #       /|\       This is also the order that you would get if
        #     *C E F      you enqueued changes in the order ABCDEFG, so
        #     /           the ordering is stable across re-enqueue events.
        #    D

        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
        C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
        D = self.fake_gerrit.addFakeChange('org/project', 'master', 'D')
        E = self.fake_gerrit.addFakeChange('org/project', 'master', 'E')
        F = self.fake_gerrit.addFakeChange('org/project', 'master', 'F')
        G = self.fake_gerrit.addFakeChange('org/project', 'master', 'G')
        B.setDependsOn(A, 1)
        C.setDependsOn(B, 1)
        D.setDependsOn(C, 1)
        E.setDependsOn(B, 1)
        F.setDependsOn(B, 1)
        G.setDependsOn(A, 1)

        A.addApproval('Code-Review', 2)
        B.addApproval('Code-Review', 2)
        C.addApproval('Code-Review', 2)
        D.addApproval('Code-Review', 2)
        E.addApproval('Code-Review', 2)
        F.addApproval('Code-Review', 2)
        G.addApproval('Code-Review', 2)
        self.fake_gerrit.addEvent(C.addApproval('Approved', 1))

        self.waitUntilSettled()

        self.assertEqual(A.data['status'], 'NEW')
        self.assertEqual(B.data['status'], 'NEW')
        self.assertEqual(C.data['status'], 'NEW')
        self.assertEqual(D.data['status'], 'NEW')
        self.assertEqual(E.data['status'], 'NEW')
        self.assertEqual(F.data['status'], 'NEW')
        self.assertEqual(G.data['status'], 'NEW')

        # We're about to add approvals to changes without adding the
        # triggering events to Zuul, so that we can be sure that it is
        # enqueing the changes based on dependencies, not because of
        # triggering events.  Since it will have the changes cached
        # already (without approvals), we need to clear the cache
        # first.
        for connection in self.scheds.first.connections.connections.values():
            if hasattr(connection, '_change_cache'):
                connection.maintainCache([], max_age=0)

        self.executor_server.hold_jobs_in_build = True
        A.addApproval('Approved', 1)
        B.addApproval('Approved', 1)
        D.addApproval('Approved', 1)
        E.addApproval('Approved', 1)
        F.addApproval('Approved', 1)
        G.addApproval('Approved', 1)
        self.fake_gerrit.addEvent(C.addApproval('Approved', 1))

        for x in range(8):
            self.executor_server.release('.*-merge')
            self.waitUntilSettled()
        self.executor_server.hold_jobs_in_build = False
        self.executor_server.release()
        self.waitUntilSettled()

        self.assertEqual(A.data['status'], 'MERGED')
        self.assertEqual(B.data['status'], 'MERGED')
        self.assertEqual(C.data['status'], 'MERGED')
        self.assertEqual(D.data['status'], 'MERGED')
        self.assertEqual(E.data['status'], 'MERGED')
        self.assertEqual(F.data['status'], 'MERGED')
        self.assertEqual(G.data['status'], 'MERGED')
        self.assertEqual(A.reported, 2)
        self.assertEqual(B.reported, 2)
        self.assertEqual(C.reported, 2)
        self.assertEqual(D.reported, 2)
        self.assertEqual(E.reported, 2)
        self.assertEqual(F.reported, 2)
        self.assertEqual(G.reported, 2)
        self.assertEqual(self.history[6].changes,
                         '1,1 2,1 3,1 4,1 5,1 6,1 7,1')

    def test_source_cache(self):
        "Test that the source cache operates correctly"
        self.executor_server.hold_jobs_in_build = True

        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
        X = self.fake_gerrit.addFakeChange('org/project', 'master', 'X')
        A.addApproval('Code-Review', 2)
        B.addApproval('Code-Review', 2)

        M1 = self.fake_gerrit.addFakeChange('org/project', 'master', 'M1')
        M1.setMerged()

        B.setDependsOn(A, 1)
        A.setDependsOn(M1, 1)

        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.fake_gerrit.addEvent(X.getPatchsetCreatedEvent(1))

        self.waitUntilSettled()

        for build in self.builds:
            if build.pipeline == 'check':
                build.release()
        self.waitUntilSettled()
        for build in self.builds:
            if build.pipeline == 'check':
                build.release()
        self.waitUntilSettled()

        self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
        self.waitUntilSettled()

        cached_changes = list(self.fake_gerrit._change_cache)
        self.log.debug("len %s", [c.cache_key for c in cached_changes])
        # there should still be changes in the cache
        self.assertNotEqual(len(cached_changes), 0)

        self.executor_server.hold_jobs_in_build = False
        self.executor_server.release()
        self.waitUntilSettled()

        self.assertEqual(A.data['status'], 'MERGED')
        self.assertEqual(B.data['status'], 'MERGED')
        self.assertEqual(A.queried, 2)  # Initial and isMerged
        self.assertEqual(B.queried, 3)  # Initial A, refresh from B, isMerged

    def test_connection_cache_cleanup(self):
        "Test that cached changes are correctly cleaned up"
        sched = self.scheds.first.sched

        def _getCachedChanges():
            cached = set()
            for source in sched.connections.getSources():
                cached.update(source.getCachedChanges())
            return cached

        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
        B.setDependsOn(A, 1)
        self.hold_jobs_in_queue = True
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
        C.setMerged()
        self.fake_gerrit.addEvent(C.getRefUpdatedEvent())
        self.waitUntilSettled()

        self.assertEqual(len(_getCachedChanges()), 3)
        sched.maintainConnectionCache()
        self.assertEqual(len(_getCachedChanges()), 3)

        # Test this method separately to make sure we are getting
        # cache keys of the correct type, since we don't do run-time
        # validation.
        relevant = sched._gatherConnectionCacheKeys()
        self.assertEqual(len(relevant), 2)
        for k in relevant:
            if not isinstance(k, ChangeKey):
                raise RuntimeError("Cache key %s is not a ChangeKey" % repr(k))

        self.hold_jobs_in_queue = False
        self.executor_api.release()
        self.waitUntilSettled()

        self.assertEqual(len(_getCachedChanges()), 3)
        sched.maintainConnectionCache()
        self.assertEqual(len(_getCachedChanges()), 3)

        # Test that outdated but still relevant changes are not cleaned up
        for connection in sched.connections.connections.values():
            connection.maintainCache(
                set([c.cache_stat.key for c in _getCachedChanges()]),
                max_age=0)
        self.assertEqual(len(_getCachedChanges()), 3)

        change1 = None
        change2 = None
        for c in _getCachedChanges():
            if c.cache_stat.key.stable_id == '1':
                change1 = c
            if c.cache_stat.key.stable_id == '2':
                change2 = c
        # Make change1 eligible for cleanup, but not change2
        change1.cache_stat = zuul.model.CacheStat(change1.cache_stat.key,
                                                  change1.cache_stat.uuid,
                                                  change1.cache_stat.version,
                                                  change1.cache_stat.mzxid,
                                                  0.0, 0, 0)
        # We should not delete change1 since it's needed by change2
        # which we want to keep.
        for connection in sched.connections.connections.values():
            connection.maintainCache([], max_age=7200)
        self.assertEqual(len(_getCachedChanges()), 3)

        # Make both changes eligible for deletion
        change2.cache_stat = zuul.model.CacheStat(change2.cache_stat.key,
                                                  change2.cache_stat.uuid,
                                                  change2.cache_stat.version,
                                                  change1.cache_stat.mzxid,
                                                  0.0, 0, 0)
        for connection in sched.connections.connections.values():
            connection.maintainCache([], max_age=7200)
        # The master branch change remains
        self.assertEqual(len(_getCachedChanges()), 1)

        # Test that we can remove changes once max_age has expired
        for connection in sched.connections.connections.values():
            connection.maintainCache([], max_age=0)
        self.assertEqual(len(_getCachedChanges()), 0)

    def test_general_cleanup(self):
        # Exercise the general cleanup method
        self.scheds.first.sched._runGeneralCleanup()

    def test_config_cache_cleanup(self):
        old_cached_projects = set(
            self.scheds.first.sched.unparsed_config_cache.listCachedProjects())
        # TODO: The only way the config_object_cache can get smaller
        # is if the cleanup is run by a newly (re-)started scheduler.
        # The unparsed_config_cache cleanup is based on that,
        # therefore that is the only way for it to get smaller as
        # well.  This could potentially be improved.  If this is
        # changed, remove the explicit clear call below (it simulates
        # a restarted scheduler).
        self.scheds.first.sched.abide.config_object_cache.clear()
        self.newTenantConfig('config/single-tenant/main-one-project.yaml')
        self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
        self.waitUntilSettled()
        self.scheds.first.sched._runGeneralCleanup()
        new_cached_projects = set(
            self.scheds.first.sched.unparsed_config_cache.listCachedProjects())
        self.assertTrue(new_cached_projects < old_cached_projects)

    def test_can_merge(self):
        "Test whether a change is ready to merge"
        # TODO: move to test_gerrit (this is a unit test!)
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
        (trusted, project) = tenant.getProject('org/project')
        source = project.source

        # TODO(pabelanger): As we add more source / trigger APIs we should make
        # it easier for users to create events for testing.
        event = zuul.model.TriggerEvent()
        event.trigger_name = 'gerrit'
        event.change_number = '1'
        event.patch_number = '2'

        a = source.getChange(source.getChangeKey(event), event=event)
        mgr = tenant.layout.pipeline_managers['gate']
        self.assertFalse(source.canMerge(a, mgr.getSubmitAllowNeeds()))

        A.addApproval('Code-Review', 2)
        a = source.getChange(source.getChangeKey(event),
                             refresh=True, event=event)
        self.assertFalse(source.canMerge(a, mgr.getSubmitAllowNeeds()))

        A.addApproval('Approved', 1)
        a = source.getChange(source.getChangeKey(event),
                             refresh=True, event=event)
        self.assertTrue(source.canMerge(a, mgr.getSubmitAllowNeeds()))

        A.setWorkInProgress(True)
        a = source.getChange(source.getChangeKey(event),
                             refresh=True, event=event)
        self.assertFalse(source.canMerge(a, mgr.getSubmitAllowNeeds()))

    def test_project_merge_conflict(self):
        "Test that gate merge conflicts are handled properly"

        self.hold_jobs_in_queue = True
        A = self.fake_gerrit.addFakeChange('org/project',
                                           'master', 'A',
                                           files={'conflict': 'foo'})
        B = self.fake_gerrit.addFakeChange('org/project',
                                           'master', 'B',
                                           files={'conflict': 'bar'})
        C = self.fake_gerrit.addFakeChange('org/project',
                                           'master', 'C')
        A.addApproval('Code-Review', 2)
        B.addApproval('Code-Review', 2)
        C.addApproval('Code-Review', 2)
        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
        self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
        self.waitUntilSettled()

        self.assertEqual(A.reported, 1)
        self.assertEqual(C.reported, 1)

        self.executor_api.release('project-merge')
        self.waitUntilSettled()
        self.executor_api.release('project-merge')
        self.waitUntilSettled()
        self.executor_api.release('project-merge')
        self.waitUntilSettled()

        self.hold_jobs_in_queue = False
        self.executor_api.release()
        self.waitUntilSettled()

        self.assertEqual(A.data['status'], 'MERGED')
        self.assertEqual(B.data['status'], 'NEW')
        self.assertEqual(C.data['status'], 'MERGED')
        self.assertEqual(A.reported, 2)
        self.assertIn('Merge Failed', B.messages[-1])
        self.assertEqual(C.reported, 2)

        self.assertHistory([
            dict(name='project-merge', result='SUCCESS', changes='1,1'),
            dict(name='project-test1', result='SUCCESS', changes='1,1'),
            dict(name='project-test2', result='SUCCESS', changes='1,1'),
            dict(name='project-merge', result='SUCCESS', changes='1,1 3,1'),
            dict(name='project-test1', result='SUCCESS', changes='1,1 3,1'),
            dict(name='project-test2', result='SUCCESS', changes='1,1 3,1'),
        ], ordered=False)

    def test_delayed_merge_conflict(self):
        "Test that delayed check merge conflicts are handled properly"

        # Hold jobs in the ZooKeeper queue so that we can test whether
        # the executor sucesfully merges a change based on an old
        # repo state (frozen by the scheduler) which would otherwise
        # conflict.
        self.hold_jobs_in_queue = True
        A = self.fake_gerrit.addFakeChange('org/project',
                                           'master', 'A',
                                           files={'conflict': 'foo'})
        B = self.fake_gerrit.addFakeChange('org/project',
                                           'master', 'B',
                                           files={'conflict': 'bar'})
        C = self.fake_gerrit.addFakeChange('org/project',
                                           'master', 'C')
        C.setDependsOn(B, 1)

        # A enters the gate queue; B and C enter the check queue
        A.addApproval('Code-Review', 2)
        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.waitUntilSettled()
        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()
        self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        self.assertEqual(A.reported, 1)
        self.assertEqual(B.reported, 0)  # Check does not report start
        self.assertEqual(C.reported, 0)  # Check does not report start

        # A merges while B and C are queued in check
        # Release A project-merge
        queue = list(self.executor_api.queued())
        self.executor_api.release(queue[0])
        self.waitUntilSettled()

        # Release A project-test*
        # gate has higher precedence, so A's test jobs are added in
        # front of the merge jobs for B and C
        queue = list(self.executor_api.queued())
        self.executor_api.release(queue[0])
        self.executor_api.release(queue[1])
        self.waitUntilSettled()

        self.assertEqual(A.data['status'], 'MERGED')
        self.assertEqual(B.data['status'], 'NEW')
        self.assertEqual(C.data['status'], 'NEW')
        self.assertEqual(A.reported, 2)
        self.assertEqual(B.reported, 0)
        self.assertEqual(C.reported, 0)
        self.assertHistory([
            dict(name='project-merge', result='SUCCESS', changes='1,1'),
            dict(name='project-test1', result='SUCCESS', changes='1,1'),
            dict(name='project-test2', result='SUCCESS', changes='1,1'),
        ], ordered=False)

        # B and C report merge conflicts
        # Release B project-merge
        queue = list(self.executor_api.queued())
        self.executor_api.release(queue[0])
        self.waitUntilSettled()

        # Release C
        self.hold_jobs_in_queue = False
        self.executor_api.release()
        self.waitUntilSettled()

        self.assertEqual(A.data['status'], 'MERGED')
        self.assertEqual(B.data['status'], 'NEW')
        self.assertEqual(C.data['status'], 'NEW')
        self.assertEqual(A.reported, 2)
        self.assertEqual(B.reported, 1)
        self.assertEqual(C.reported, 1)

        self.assertHistory([
            dict(name='project-merge', result='SUCCESS', changes='1,1'),
            dict(name='project-test1', result='SUCCESS', changes='1,1'),
            dict(name='project-test2', result='SUCCESS', changes='1,1'),
            dict(name='project-merge', result='SUCCESS', changes='2,1'),
            dict(name='project-test1', result='SUCCESS', changes='2,1'),
            dict(name='project-test2', result='SUCCESS', changes='2,1'),
            dict(name='project-merge', result='SUCCESS', changes='2,1 3,1'),
            dict(name='project-test1', result='SUCCESS', changes='2,1 3,1'),
            dict(name='project-test2', result='SUCCESS', changes='2,1 3,1'),
        ], ordered=False)

    def test_post(self):
        "Test that post jobs run"
        p = "review.example.com/org/project"
        upstream = self.getUpstreamRepos([p])
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        A.setMerged()
        A_commit = str(upstream[p].commit('master'))
        self.log.debug("A commit: %s" % A_commit)

        e = {
            "type": "ref-updated",
            "submitter": {
                "name": "User Name",
            },
            "refUpdate": {
                "oldRev": "90f173846e3af9154517b88543ffbd1691f31366",
                "newRev": A_commit,
                "refName": "master",
                "project": "org/project",
            }
        }
        self.fake_gerrit.addEvent(e)
        self.waitUntilSettled()

        job_names = [x.name for x in self.history]
        self.assertEqual(len(self.history), 1)
        self.assertIn('project-post', job_names)

    @simple_layout('layouts/negate-post.yaml')
    def test_post_negative_regex(self):
        "Test that post jobs run"
        p = "review.example.com/org/project"
        upstream = self.getUpstreamRepos([p])
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        A.setMerged()
        A_commit = str(upstream[p].commit('master'))
        self.log.debug("A commit: %s" % A_commit)

        e = {
            "type": "ref-updated",
            "submitter": {
                "name": "User Name",
            },
            "refUpdate": {
                "oldRev": "90f173846e3af9154517b88543ffbd1691f31366",
                "newRev": A_commit,
                "refName": "master",
                "project": "org/project",
            }
        }
        self.fake_gerrit.addEvent(e)
        self.waitUntilSettled()

        job_names = [x.name for x in self.history]
        self.assertEqual(len(self.history), 1)
        self.assertIn('post-job', job_names)

    def test_post_ignore_deletes(self):
        "Test that deleting refs does not trigger post jobs"

        e = {
            "type": "ref-updated",
            "submitter": {
                "name": "User Name",
            },
            "refUpdate": {
                "oldRev": "90f173846e3af9154517b88543ffbd1691f31366",
                "newRev": "0000000000000000000000000000000000000000",
                "refName": "master",
                "project": "org/project",
            }
        }
        self.fake_gerrit.addEvent(e)
        self.waitUntilSettled()

        job_names = [x.name for x in self.history]
        self.assertEqual(len(self.history), 0)
        self.assertNotIn('project-post', job_names)

    @simple_layout('layouts/dont-ignore-ref-deletes.yaml')
    def test_post_ignore_deletes_negative(self):
        "Test that deleting refs does trigger post jobs"
        e = {
            "type": "ref-updated",
            "submitter": {
                "name": "User Name",
            },
            "refUpdate": {
                "oldRev": "90f173846e3af9154517b88543ffbd1691f31366",
                "newRev": "0000000000000000000000000000000000000000",
                "refName": "testbranch",
                "project": "org/project",
            }
        }
        self.fake_gerrit.addEvent(e)
        self.waitUntilSettled()

        job_names = [x.name for x in self.history]
        self.assertEqual(len(self.history), 1)
        self.assertIn('project-post', job_names)

    @skip("Disabled for early v3 development")
    def test_build_configuration_branch_interaction(self):
        "Test that switching between branches works"
        self.test_build_configuration()
        self.test_build_configuration_branch()
        # C has been merged, undo that
        path = os.path.join(self.upstream_root, "org/project")
        repo = git.Repo(path)
        repo.heads.master.commit = repo.commit('init')
        self.test_build_configuration()

    def test_dependent_changes_rebase(self):
        # Test that no errors occur when we walk a dependency tree
        # with an unused leaf node due to a rebase.
        # Start by constructing: C -> B -> A
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
        B.setDependsOn(A, 1)

        C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
        C.setDependsOn(B, 1)

        # Then rebase to form: D -> C -> A
        C.addPatchset()  # C,2
        C.setDependsOn(A, 1)

        D = self.fake_gerrit.addFakeChange('org/project', 'master', 'D')
        D.setDependsOn(C, 2)

        # Walk the entire tree
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()
        self.assertEqual(len(self.history), 3)

        # Verify that walking just part of the tree still works
        self.fake_gerrit.addEvent(D.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()
        self.assertEqual(len(self.history), 6)

    def test_dependent_changes_dequeue(self):
        "Test that dependent patches are not needlessly tested"

        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
        C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
        A.addApproval('Code-Review', 2)
        B.addApproval('Code-Review', 2)
        C.addApproval('Code-Review', 2)

        M1 = self.fake_gerrit.addFakeChange('org/project', 'master', 'M1')
        M1.setMerged()

        # C -> B -> A -> M1

        C.setDependsOn(B, 1)
        B.setDependsOn(A, 1)
        A.setDependsOn(M1, 1)

        self.executor_server.failJob('project-merge', A)

        self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
        self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

        self.waitUntilSettled()

        self.assertEqual(A.data['status'], 'NEW')
        self.assertEqual(A.reported, 2)
        self.assertEqual(B.data['status'], 'NEW')
        self.assertEqual(B.reported, 2)
        self.assertEqual(C.data['status'], 'NEW')
        self.assertIn('This change depends on a change that failed to merge.',
                      C.messages[-1])
        self.assertEqual(len(self.history), 1)

    def test_failing_dependent_changes(self):
        "Test that failing dependent patches are taken out of stream"
        self.executor_server.hold_jobs_in_build = True
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
        C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
        D = self.fake_gerrit.addFakeChange('org/project', 'master', 'D')
        E = self.fake_gerrit.addFakeChange('org/project', 'master', 'E')
        A.addApproval('Code-Review', 2)
        B.addApproval('Code-Review', 2)
        C.addApproval('Code-Review', 2)
        D.addApproval('Code-Review', 2)
        E.addApproval('Code-Review', 2)

        # E, D -> C -> B, A

        D.setDependsOn(C, 1)
        C.setDependsOn(B, 1)

        self.executor_server.failJob('project-test1', B)

        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.fake_gerrit.addEvent(D.addApproval('Approved', 1))
        self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
        self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
        self.fake_gerrit.addEvent(E.addApproval('Approved', 1))

        self.waitUntilSettled()
        self.executor_server.release('.*-merge')
        self.waitUntilSettled()
        self.executor_server.release('.*-merge')
        self.waitUntilSettled()
        self.executor_server.release('.*-merge')
        self.waitUntilSettled()
        self.executor_server.release('.*-merge')
        self.waitUntilSettled()
        self.executor_server.release('.*-merge')
        self.waitUntilSettled()

        self.executor_server.hold_jobs_in_build = False
        for build in self.builds:
            if build.parameters['zuul']['change'] != '1':
                build.release()
                self.waitUntilSettled()

        self.executor_server.release()
        self.waitUntilSettled()

        self.assertEqual(A.data['status'], 'MERGED')
        self.assertEqual(A.reported, 2)
        self.assertIn('Build succeeded', A.messages[1])
        self.assertEqual(B.data['status'], 'NEW')
        self.assertEqual(B.reported, 2)
        self.assertIn('Build failed', B.messages[1])
        self.assertEqual(C.data['status'], 'NEW')
        self.assertEqual(C.reported, 2)
        self.assertIn('depends on a change', C.messages[1])
        self.assertEqual(D.data['status'], 'NEW')
        self.assertEqual(D.reported, 2)
        self.assertIn('depends on a change', D.messages[1])
        self.assertEqual(E.data['status'], 'MERGED')
        self.assertEqual(E.reported, 2)
        self.assertIn('Build succeeded', E.messages[1])
        self.assertEqual(len(self.history), 18)

    def test_head_is_dequeued_once(self):
        "Test that if a change at the head fails it is dequeued only once"
        # If it's dequeued more than once, we should see extra
        # aborted jobs.

        self.executor_server.hold_jobs_in_build = True
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
        C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
        A.addApproval('Code-Review', 2)
        B.addApproval('Code-Review', 2)
        C.addApproval('Code-Review', 2)

        self.executor_server.failJob('project-test1', A)
        self.executor_server.failJob('project-test2', A)

        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
        self.fake_gerrit.addEvent(C.addApproval('Approved', 1))

        self.waitUntilSettled()

        self.assertEqual(len(self.builds), 1)
        self.assertEqual(self.builds[0].name, 'project-merge')
        self.assertTrue(self.builds[0].hasChanges(A))

        self.executor_server.release('.*-merge')
        self.waitUntilSettled()
        self.executor_server.release('.*-merge')
        self.waitUntilSettled()
        self.executor_server.release('.*-merge')
        self.waitUntilSettled()

        self.assertEqual(len(self.builds), 6)
        self.assertEqual(self.builds[0].name, 'project-test1')
        self.assertEqual(self.builds[1].name, 'project-test2')
        self.assertEqual(self.builds[2].name, 'project-test1')
        self.assertEqual(self.builds[3].name, 'project-test2')
        self.assertEqual(self.builds[4].name, 'project-test1')
        self.assertEqual(self.builds[5].name, 'project-test2')

        self.release(self.builds[0])
        self.waitUntilSettled()

        self.assertEqual(len(self.builds), 2)  # test2, merge for B
        self.assertEqual(self.countJobResults(self.history, 'ABORTED'), 4)

        self.executor_server.hold_jobs_in_build = False
        self.executor_server.release()
        self.waitUntilSettled()

        self.assertEqual(len(self.builds), 0)
        self.assertEqual(len(self.history), 15)

        self.assertEqual(A.data['status'], 'NEW')
        self.assertEqual(B.data['status'], 'MERGED')
        self.assertEqual(C.data['status'], 'MERGED')
        self.assertEqual(A.reported, 2)
        self.assertEqual(B.reported, 2)
        self.assertEqual(C.reported, 2)

    def test_dequeue_only_live(self):
        # Test that we only dequeue live items
        self.executor_server.hold_jobs_in_build = True
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
        B.setDependsOn(A, 1)

        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        self.scheds.first.sched.dequeue(
            "tenant-one", "check", "org/project", "1,1", None)

        self.executor_server.hold_jobs_in_build = False
        self.executor_server.release()
        self.waitUntilSettled()

        self.assertHistory([
            dict(name='project-merge', result='ABORTED', changes='1,1'),
            dict(name='project-merge', result='SUCCESS', changes='1,1 2,1'),
            dict(name='project-test1', result='SUCCESS', changes='1,1 2,1'),
            dict(name='project-test2', result='SUCCESS', changes='1,1 2,1'),
        ], ordered=False)

    def test_approval_removal(self):
        # Test that we dequeue a change when it can not merge
        self.executor_server.hold_jobs_in_build = True
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        A.addApproval('Code-Review', 2)
        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

        self.waitUntilSettled()
        self.assertEqual(1, len(self.builds))
        self.assertEqual(0, len(self.history))

        # Remove the approval
        self.fake_gerrit.addEvent(A.addApproval('Approved', 0))
        self.waitUntilSettled()

        self.executor_server.hold_jobs_in_build = False
        self.executor_server.release()
        self.waitUntilSettled()

        # The change should be dequeued.
        self.assertHistory([
            dict(name='project-merge', result='ABORTED'),
        ], ordered=False)
        self.assertEqual(2, len(A.messages))
        self.assertEqual(A.data['status'], 'NEW')
        self.assertIn('This change is unable to merge '
                      'due to a missing merge requirement.',
                      A.messages[1])

    @simple_layout('layouts/nonvoting-job-approval.yaml')
    def test_nonvoting_job_approval(self):
        "Test that non-voting jobs don't vote but leave approval"

        A = self.fake_gerrit.addFakeChange('org/nonvoting-project',
                                           'master', 'A')
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.executor_server.failJob('nonvoting-project-test2', A)

        self.waitUntilSettled()

        self.assertEqual(A.data['status'], 'NEW')
        self.assertEqual(A.reported, 1)

        self.assertEqual(
            self.getJobFromHistory('nonvoting-project-test1').result,
            'SUCCESS')
        self.assertEqual(
            self.getJobFromHistory('nonvoting-project-test2').result,
            'FAILURE')

        self.assertFalse(self.getJobFromHistory('nonvoting-project-test1').
                         parameters['zuul']['voting'])
        self.assertFalse(self.getJobFromHistory('nonvoting-project-test2').
                         parameters['zuul']['voting'])
        self.assertEqual(A.patchsets[0]['approvals'][0]['value'], "1")

    @simple_layout('layouts/nonvoting-job.yaml')
    def test_nonvoting_job(self):
        "Test that non-voting jobs don't vote."

        A = self.fake_gerrit.addFakeChange('org/nonvoting-project',
                                           'master', 'A')
        A.addApproval('Code-Review', 2)
        self.executor_server.failJob('nonvoting-project-test2', A)
        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

        self.waitUntilSettled()

        self.assertEqual(A.data['status'], 'MERGED')
        self.assertEqual(A.reported, 2)
        self.assertEqual(
            self.getJobFromHistory('nonvoting-project-merge').result,
            'SUCCESS')
        self.assertEqual(
            self.getJobFromHistory('nonvoting-project-test1').result,
            'SUCCESS')
        self.assertEqual(
            self.getJobFromHistory('nonvoting-project-test2').result,
            'FAILURE')

        self.assertTrue(self.getJobFromHistory('nonvoting-project-merge').
                        parameters['zuul']['voting'])
        self.assertTrue(self.getJobFromHistory('nonvoting-project-test1').
                        parameters['zuul']['voting'])
        self.assertFalse(self.getJobFromHistory('nonvoting-project-test2').
                         parameters['zuul']['voting'])

    def test_check_queue_success(self):
        "Test successful check queue jobs."

        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

        self.waitUntilSettled()

        self.assertEqual(A.data['status'], 'NEW')
        self.assertEqual(A.reported, 1)
        self.assertEqual(self.getJobFromHistory('project-merge').result,
                         'SUCCESS')
        self.assertEqual(self.getJobFromHistory('project-test1').result,
                         'SUCCESS')
        self.assertEqual(self.getJobFromHistory('project-test2').result,
                         'SUCCESS')

    def test_check_queue_failure(self):
        "Test failed check queue jobs."

        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        self.executor_server.failJob('project-test2', A)
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

        self.waitUntilSettled()

        self.assertEqual(A.data['status'], 'NEW')
        self.assertEqual(A.reported, 1)
        self.assertEqual(self.getJobFromHistory('project-merge').result,
                         'SUCCESS')
        self.assertEqual(self.getJobFromHistory('project-test1').result,
                         'SUCCESS')
        self.assertEqual(self.getJobFromHistory('project-test2').result,
                         'FAILURE')

    @simple_layout('layouts/autohold.yaml')
    def test_autohold(self):
        self.scheds.first.sched.autohold(
            'tenant-one', 'review.example.com/org/project', 'project-test2',
            ".*", "reason text", 1, None)

        # There should be a record in ZooKeeper
        request_list = self.sched_zk_nodepool.getHoldRequests()
        self.assertEqual(1, len(request_list))
        request = self.sched_zk_nodepool.getHoldRequest(
            request_list[0])
        self.assertIsNotNone(request)
        self.assertEqual('tenant-one', request.tenant)
        self.assertEqual('review.example.com/org/project', request.project)
        self.assertEqual('project-test2', request.job)
        self.assertEqual('reason text', request.reason)
        self.assertEqual(1, request.max_count)
        self.assertEqual(0, request.current_count)
        self.assertEqual([], request.nodes)

        # Some convenience variables for checking the stats.
        tenant_ram_stat =\
            'zuul.nodepool.resources.in_use.tenant.tenant-one.ram'
        project_ram_stat = ('zuul.nodepool.resources.in_use.project.'
                            'review_example_com/org/project.ram')
        # Test that we zeroed the gauges
        self.scheds.first.sched._runStats()
        self.assertUnReportedStat(tenant_ram_stat, value='1024', kind='g')
        self.assertUnReportedStat(project_ram_stat, value='1024', kind='g')
        self.assertReportedStat(tenant_ram_stat, value='0', kind='g')
        self.assertReportedStat(project_ram_stat, value='0', kind='g')

        # First check that successful jobs do not autohold
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

        self.waitUntilSettled()

        self.assertEqual(A.data['status'], 'NEW')
        self.assertEqual(A.reported, 1)
        # project-test2
        self.assertEqual(self.history[0].result, 'SUCCESS')

        # Check nodepool for a held node
        held_node = None
        for node in self.fake_nodepool.getNodes():
            if node['state'] == zuul.model.STATE_HOLD:
                held_node = node
                break
        self.assertIsNone(held_node)

        # Hold in build to check the stats
        self.executor_server.hold_jobs_in_build = True

        # Now test that failed jobs are autoheld

        # Set resources only for this node so we can examine the code
        # path for updating the stats on autohold.
        self.fake_nodepool.resources = {
            'cores': 2,
            'ram': 1024,
            'instances': 1,
        }

        B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
        self.executor_server.failJob('project-test2', B)
        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))

        self.waitUntilSettled()

        # Get the build request object
        build = list(self.getCurrentBuilds())[0]

        # We should report using the held node's resources
        self.waitUntilNodeCacheSync(
            self.scheds.first.sched.nodepool.zk_nodepool)
        self.statsd.clear()
        self.scheds.first.sched._runStats()
        self.assertReportedStat(tenant_ram_stat, value='1024', kind='g')
        self.assertReportedStat(project_ram_stat, value='1024', kind='g')
        self.assertUnReportedStat(tenant_ram_stat, value='0', kind='g')
        self.assertUnReportedStat(project_ram_stat, value='0', kind='g')

        self.executor_server.hold_jobs_in_build = False
        self.executor_server.release()
        self.waitUntilSettled()

        self.assertEqual(B.data['status'], 'NEW')
        self.assertEqual(B.reported, 1)
        # project-test2
        self.assertEqual(self.history[1].result, 'FAILURE')
        self.assertTrue(build.held)

        # Check nodepool for a held node
        held_node = None
        for node in self.fake_nodepool.getNodes():
            if node['state'] == zuul.model.STATE_HOLD:
                held_node = node
                break
        self.assertIsNotNone(held_node)
        # Validate node has recorded the failed job
        self.assertEqual(
            held_node['hold_job'],
            " ".join(['tenant-one',
                      'review.example.com/org/project',
                      'project-test2', '.*'])
        )
        self.assertEqual(held_node['comment'], "reason text")

        # The hold request current_count should have incremented
        # and we should have recorded the held node ID.
        request2 = self.sched_zk_nodepool.getHoldRequest(
            request.id)
        self.assertEqual(request.current_count + 1, request2.current_count)
        self.assertEqual(1, len(request2.nodes))
        self.assertEqual(1, len(request2.nodes[0]["nodes"]))

        # We should still report we use the resources
        self.waitUntilNodeCacheSync(
            self.scheds.first.sched.nodepool.zk_nodepool)
        self.statsd.clear()
        self.scheds.first.sched._runStats()
        self.assertReportedStat(tenant_ram_stat, value='1024', kind='g')
        self.assertReportedStat(project_ram_stat, value='1024', kind='g')
        self.assertUnReportedStat(tenant_ram_stat, value='0', kind='g')
        self.assertUnReportedStat(project_ram_stat, value='0', kind='g')

        # Another failed change should not hold any more nodes
        self.fake_nodepool.resources = {}
        C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
        self.executor_server.failJob('project-test2', C)
        self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()
        self.assertEqual(C.data['status'], 'NEW')
        self.assertEqual(C.reported, 1)
        # project-test2
        self.assertEqual(self.history[2].result, 'FAILURE')

        held_nodes = 0
        for node in self.fake_nodepool.getNodes():
            if node['state'] == zuul.model.STATE_HOLD:
                held_nodes += 1
        self.assertEqual(held_nodes, 1)

        # request current_count should not have changed
        request3 = self.sched_zk_nodepool.getHoldRequest(
            request2.id)
        self.assertEqual(request2.current_count, request3.current_count)

        # Deleting hold request should set held nodes to used
        self.sched_zk_nodepool.deleteHoldRequest(request3, None)
        node_states = [n['state'] for n in self.fake_nodepool.getNodes()]
        self.assertEqual(3, len(node_states))
        self.assertEqual([zuul.model.STATE_USED] * 3, node_states)

        # Nodepool deletes the nodes
        for n in self.fake_nodepool.getNodes():
            self.fake_nodepool.removeNode(n)

        # We should now report that we no longer use the nodes resources
        self.waitUntilNodeCacheSync(
            self.scheds.first.sched.nodepool.zk_nodepool)
        self.statsd.clear()
        self.scheds.first.sched._runStats()
        self.assertUnReportedStat(tenant_ram_stat, value='1024', kind='g')
        self.assertUnReportedStat(project_ram_stat, value='1024', kind='g')
        self.assertReportedStat(tenant_ram_stat, value='0', kind='g')
        self.assertReportedStat(project_ram_stat, value='0', kind='g')

    @simple_layout('layouts/autohold.yaml')
    def test_autohold_info(self):
        self.scheds.first.sched.autohold(
            'tenant-one', 'review.example.com/org/project', 'project-test2',
            ".*", "reason text", 1, None)
        # There should be a record in ZooKeeper
        request_list = self.sched_zk_nodepool.getHoldRequests()
        self.assertEqual(1, len(request_list))
        request = self.sched_zk_nodepool.getHoldRequest(
            request_list[0])
        self.assertIsNotNone(request)

        request = self.scheds.first.sched.autohold_info(request.id)
        self.assertNotEqual({}, request)
        self.assertEqual('tenant-one', request['tenant'])
        self.assertEqual('review.example.com/org/project', request['project'])
        self.assertEqual('project-test2', request['job'])
        self.assertEqual('reason text', request['reason'])
        self.assertEqual(1, request['max_count'])
        self.assertEqual(0, request['current_count'])

    @simple_layout('layouts/autohold.yaml')
    def test_autohold_delete(self):
        self.scheds.first.sched.autohold(
            'tenant-one', 'review.example.com/org/project', 'project-test2',
            ".*", "reason text", 1, None)

        # There should be a record in ZooKeeper
        request_list = self.sched_zk_nodepool.getHoldRequests()
        self.assertEqual(1, len(request_list))
        request = self.sched_zk_nodepool.getHoldRequest(
            request_list[0])
        self.assertIsNotNone(request)

        # Delete and verify no more requests
        self.scheds.first.sched.autohold_delete(request.id)
        request_list = self.sched_zk_nodepool.getHoldRequests()
        self.assertEqual([], request_list)

    def test_autohold_padding(self):
        self.scheds.first.sched.autohold(
            'tenant-one', 'review.example.com/org/project', 'project-test2',
            ".*", "reason text", 1, None)

        # There should be a record in ZooKeeper
        request_list = self.sched_zk_nodepool.getHoldRequests()
        self.assertEqual(1, len(request_list))
        request = self.sched_zk_nodepool.getHoldRequest(
            request_list[0])
        self.assertIsNotNone(request)

        # Assert the ID leads with a bunch of zeros, them strip them
        # off to test autohold_delete can handle a user passing in an
        # ID without leading zeros.
        self.assertEqual(request.id[0:5], '00000')
        trimmed_request = request.id[5:]
        # Delete and verify no more requests
        self.scheds.first.sched.autohold_delete(trimmed_request)
        request_list = self.sched_zk_nodepool.getHoldRequests()
        self.assertEqual([], request_list)

    def _test_autohold_scoped(self, change_obj, change, ref):
        # create two changes on the same project, and autohold request
        # for one of them.
        other = self.fake_gerrit.addFakeChange(
            'org/project', 'master', 'other'
        )

        if change != "":
            ref = "refs/changes/%s/%s/.*" % (
                str(change_obj.number).zfill(2)[-2:], str(change_obj.number)
            )
        self.scheds.first.sched.autohold(
            'tenant-one', 'review.example.com/org/project', 'project-test2',
            ref, "reason text", 1, None)

        # First, check that an unrelated job does not trigger autohold, even
        # when it failed
        self.executor_server.failJob('project-test2', other)
        self.fake_gerrit.addEvent(other.getPatchsetCreatedEvent(1))

        self.waitUntilSettled()

        self.assertEqual(other.data['status'], 'NEW')
        self.assertEqual(other.reported, 1)
        # project-test2
        self.assertEqual(self.history[0].result, 'FAILURE')

        # Check nodepool for a held node
        held_node = None
        for node in self.fake_nodepool.getNodes():
            if node['state'] == zuul.model.STATE_HOLD:
                held_node = node
                break
        self.assertIsNone(held_node)

        # And then verify that failed job for the defined change
        # triggers the autohold

        self.executor_server.failJob('project-test2', change_obj)
        self.fake_gerrit.addEvent(change_obj.getPatchsetCreatedEvent(1))

        self.waitUntilSettled()

        self.assertEqual(change_obj.data['status'], 'NEW')
        self.assertEqual(change_obj.reported, 1)
        # project-test2
        self.assertEqual(self.history[1].result, 'FAILURE')

        # Check nodepool for a held node
        held_node = None
        for node in self.fake_nodepool.getNodes():
            if node['state'] == zuul.model.STATE_HOLD:
                held_node = node
                break
        self.assertIsNotNone(held_node)

        # Validate node has recorded the failed job
        self.assertEqual(
            held_node['hold_job'],
            " ".join(['tenant-one',
                      'review.example.com/org/project',
                      'project-test2', ref])
        )
        self.assertEqual(held_node['comment'], "reason text")

    @simple_layout('layouts/autohold.yaml')
    def test_autohold_change(self):
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

        self._test_autohold_scoped(A, change=A.number, ref="")

    @simple_layout('layouts/autohold.yaml')
    def test_autohold_ref(self):
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        ref = A.data['currentPatchSet']['ref']
        self._test_autohold_scoped(A, change="", ref=ref)

    @simple_layout('layouts/autohold.yaml')
    def test_autohold_scoping(self):
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

        # create three autohold requests, scoped to job, change and
        # a specific ref
        change = str(A.number)
        change_ref = "refs/changes/%s/%s/.*" % (
            str(change).zfill(2)[-2:], str(change)
        )
        ref = A.data['currentPatchSet']['ref']
        self.scheds.first.sched.autohold(
            'tenant-one', 'review.example.com/org/project', 'project-test2',
            ".*", "reason text", 1, None)
        self.scheds.first.sched.autohold(
            'tenant-one', 'review.example.com/org/project', 'project-test2',
            change_ref, "reason text", 1, None)
        self.scheds.first.sched.autohold(
            'tenant-one', 'review.example.com/org/project', 'project-test2',
            ref, "reason text", 1, None)

        # Fail 3 jobs for the same change, and verify that the autohold
        # requests are fullfilled in the expected order: from the most
        # specific towards the most generic one.

        def _fail_job_and_verify_autohold_request(change_obj, ref_filter):
            self.executor_server.failJob('project-test2', change_obj)
            self.fake_gerrit.addEvent(change_obj.getPatchsetCreatedEvent(1))

            self.waitUntilSettled()

            # Check nodepool for a held node
            held_node = None
            for node in self.fake_nodepool.getNodes():
                if node['state'] == zuul.model.STATE_HOLD:
                    held_node = node
                    break
            self.assertIsNotNone(held_node)

            self.assertEqual(
                held_node['hold_job'],
                " ".join(['tenant-one',
                          'review.example.com/org/project',
                          'project-test2', ref_filter])
            )
            self.assertFalse(held_node['_lock'], "Node %s is locked" %
                             (node['_oid'],))
            self.fake_nodepool.removeNode(held_node)

        _fail_job_and_verify_autohold_request(A, ref)

        ref = "refs/changes/%s/%s/.*" % (str(change).zfill(2)[-2:],
                                         str(change))
        _fail_job_and_verify_autohold_request(A, ref)
        _fail_job_and_verify_autohold_request(A, ".*")

    @simple_layout('layouts/autohold.yaml')
    def test_autohold_ignores_aborted_jobs(self):
        self.scheds.first.sched.autohold(
            'tenant-one', 'review.example.com/org/project', 'project-test2',
            ".*", "reason text", 1, None)

        self.executor_server.hold_jobs_in_build = True

        # Create a change that will have its job aborted
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        # Creating new patchset on change A will abort A,1's job because
        # a new patchset arrived replacing A,1 with A,2.
        A.addPatchset()
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(2))

        self.waitUntilSettled()
        self.executor_server.hold_jobs_in_build = False
        self.executor_server.release()
        self.waitUntilSettled()

        self.assertEqual(A.data['status'], 'NEW')
        # Note only the successful job for A,2 will report as we don't
        # report aborted builds for old patchsets.
        self.assertEqual(A.reported, 1)
        # A,1 project-test2
        self.assertEqual(self.history[0].result, 'ABORTED')
        # A,2 project-test2
        self.assertEqual(self.history[1].result, 'SUCCESS')

        # Check nodepool for a held node
        held_node = None
        for node in self.fake_nodepool.getNodes():
            if node['state'] == zuul.model.STATE_HOLD:
                held_node = node
                break
        self.assertIsNone(held_node)

    @simple_layout('layouts/autohold.yaml')
    def test_autohold_hold_expiration(self):
        self.scheds.first.sched.autohold(
            'tenant-one', 'review.example.com/org/project', 'project-test2',
            ".*", "reason text", 1, 30)

        # Hold a failed job
        B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
        self.executor_server.failJob('project-test2', B)
        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))

        self.waitUntilSettled()

        self.assertEqual(B.data['status'], 'NEW')
        self.assertEqual(B.reported, 1)
        # project-test2
        self.assertEqual(self.history[0].result, 'FAILURE')

        # Check nodepool for a held node
        held_node = None
        for node in self.fake_nodepool.getNodes():
            if node['state'] == zuul.model.STATE_HOLD:
                held_node = node
                break
        self.assertIsNotNone(held_node)

        # Validate node has hold_expiration property
        self.assertEqual(int(held_node['hold_expiration']), 30)

    @simple_layout('layouts/autohold.yaml')
    def test_autohold_list(self):
        self.scheds.first.sched.autohold(
            'tenant-one', 'review.example.com/org/project', 'project-test2',
            ".*", "reason text", 1, None)

        autohold_requests = self.scheds.first.sched.autohold_list()
        self.assertNotEqual([], autohold_requests)
        self.assertEqual(1, len(autohold_requests))

        request = autohold_requests[0]
        self.assertEqual('tenant-one', request['tenant'])
        self.assertIn('org/project', request['project'])
        self.assertEqual('project-test2', request['job'])
        self.assertEqual(".*", request['ref_filter'])
        self.assertEqual("reason text", request['reason'])

    @simple_layout('layouts/autohold.yaml')
    def test_autohold_request_expiration(self):
        orig_exp = RecordingExecutorServer.EXPIRED_HOLD_REQUEST_TTL

        def reset_exp():
            self.executor_server.EXPIRED_HOLD_REQUEST_TTL = orig_exp

        self.addCleanup(reset_exp)

        # Temporarily shorten the hold request expiration time
        self.scheds.first.sched.autohold(
            'tenant-one', 'review.example.com/org/project', 'project-test2',
            ".*", "reason text", 1, 1)

        autohold_requests = self.scheds.first.sched.autohold_list()
        self.assertEqual(1, len(autohold_requests))
        req = autohold_requests[0]
        self.assertIsNone(req['expired'])

        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        self.executor_server.failJob('project-test2', A)
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        autohold_requests = self.scheds.first.sched.autohold_list()
        self.assertEqual(1, len(autohold_requests))
        req = autohold_requests[0]
        self.assertIsNotNone(req['expired'])

        # Temporarily shorten hold time so that the hold request can be
        # auto-deleted (which is done on another test failure). And wait
        # long enough for nodes to expire and request to delete.
        self.executor_server.EXPIRED_HOLD_REQUEST_TTL = 1
        time.sleep(3)

        B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
        self.executor_server.failJob('project-test2', B)
        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        for _ in iterate_timeout(10, 'hold request expiration'):
            if len(self.scheds.first.sched.autohold_list()) == 0:
                break

    @simple_layout('layouts/three-projects.yaml')
    def test_dependent_behind_dequeue(self):
        # This particular test does a large amount of merges and needs a little
        # more time to complete
        self.wait_timeout = 120
        "test that dependent changes behind dequeued changes work"
        # This complicated test is a reproduction of a real life bug
        self.scheds.execute(lambda app: app.sched.reconfigure(app.config))

        self.executor_server.hold_jobs_in_build = True
        A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
        B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
        C = self.fake_gerrit.addFakeChange('org/project2', 'master', 'C')
        D = self.fake_gerrit.addFakeChange('org/project2', 'master', 'D')
        E = self.fake_gerrit.addFakeChange('org/project2', 'master', 'E')
        F = self.fake_gerrit.addFakeChange('org/project3', 'master', 'F')
        D.setDependsOn(C, 1)
        E.setDependsOn(D, 1)
        A.addApproval('Code-Review', 2)
        B.addApproval('Code-Review', 2)
        C.addApproval('Code-Review', 2)
        D.addApproval('Code-Review', 2)
        E.addApproval('Code-Review', 2)
        F.addApproval('Code-Review', 2)

        A.fail_merge = True

        # Change object re-use in the gerrit trigger is hidden if
        # changes are added in quick succession; waiting makes it more
        # like real life.
        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.waitUntilSettled()
        self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
        self.waitUntilSettled()

        self.executor_server.release('.*-merge')
        self.waitUntilSettled()
        self.executor_server.release('.*-merge')
        self.waitUntilSettled()

        self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
        self.waitUntilSettled()
        self.fake_gerrit.addEvent(D.addApproval('Approved', 1))
        self.waitUntilSettled()
        self.fake_gerrit.addEvent(E.addApproval('Approved', 1))
        self.waitUntilSettled()
        self.fake_gerrit.addEvent(F.addApproval('Approved', 1))
        self.waitUntilSettled()

        self.executor_server.release('.*-merge')
        self.waitUntilSettled()
        self.executor_server.release('.*-merge')
        self.waitUntilSettled()
        self.executor_server.release('.*-merge')
        self.waitUntilSettled()
        self.executor_server.release('.*-merge')
        self.waitUntilSettled()

        # all jobs running

        # Grab pointers to the jobs we want to release before
        # releasing any, because list indexes may change as
        # the jobs complete.
        a, b, c = self.builds[:3]
        a.release()
        b.release()
        c.release()
        self.waitUntilSettled()

        self.executor_server.hold_jobs_in_build = False
        self.executor_server.release()
        self.waitUntilSettled()

        self.assertEqual(A.data['status'], 'NEW')
        self.assertEqual(B.data['status'], 'MERGED')
        self.assertEqual(C.data['status'], 'MERGED')
        self.assertEqual(D.data['status'], 'MERGED')
        self.assertEqual(E.data['status'], 'MERGED')
        self.assertEqual(F.data['status'], 'MERGED')

        self.assertEqual(A.reported, 2)
        self.assertEqual(B.reported, 2)
        self.assertEqual(C.reported, 2)
        self.assertEqual(D.reported, 2)
        self.assertEqual(E.reported, 2)
        self.assertEqual(F.reported, 2)

        self.assertEqual(self.countJobResults(self.history, 'ABORTED'), 15)
        self.assertEqual(len(self.history), 44)

    def test_merger_repack(self):
        "Test that the merger works after a repack"

        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        A.addApproval('Code-Review', 2)
        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.waitUntilSettled()
        self.assertEqual(self.getJobFromHistory('project-merge').result,
                         'SUCCESS')
        self.assertEqual(self.getJobFromHistory('project-test1').result,
                         'SUCCESS')
        self.assertEqual(self.getJobFromHistory('project-test2').result,
                         'SUCCESS')
        self.assertEqual(A.data['status'], 'MERGED')
        self.assertEqual(A.reported, 2)
        self.assertEmptyQueues()
        self.build_history = []

        path = os.path.join(self.merger_src_root, "review.example.com",
                            "org/project")
        if os.path.exists(path):
            repack_repo(path)
        path = os.path.join(self.executor_src_root, "review.example.com",
                            "org/project")
        if os.path.exists(path):
            repack_repo(path)

        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        A.addApproval('Code-Review', 2)
        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.waitUntilSettled()
        self.assertEqual(self.getJobFromHistory('project-merge').result,
                         'SUCCESS')
        self.assertEqual(self.getJobFromHistory('project-test1').result,
                         'SUCCESS')
        self.assertEqual(self.getJobFromHistory('project-test2').result,
                         'SUCCESS')
        self.assertEqual(A.data['status'], 'MERGED')
        self.assertEqual(A.reported, 2)

    def test_merger_repack_large_change(self):
        "Test that the merger works with large changes after a repack"
        # https://bugs.executepad.net/zuul/+bug/1078946
        # This test assumes the repo is already cloned; make sure it is
        tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
        trusted, project = tenant.getProject('org/project')
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        A.addPatchset(large=True)
        # TODOv3(jeblair): add hostname to upstream root
        path = os.path.join(self.upstream_root, 'org/project')
        repack_repo(path)
        path = os.path.join(self.merger_src_root, 'review.example.com',
                            'org/project')
        if os.path.exists(path):
            repack_repo(path)
        path = os.path.join(self.executor_src_root, 'review.example.com',
                            'org/project')
        if os.path.exists(path):
            repack_repo(path)

        A.addApproval('Code-Review', 2)
        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.waitUntilSettled()
        self.assertEqual(self.getJobFromHistory('project-merge').result,
                         'SUCCESS')
        self.assertEqual(self.getJobFromHistory('project-test1').result,
                         'SUCCESS')
        self.assertEqual(self.getJobFromHistory('project-test2').result,
                         'SUCCESS')
        self.assertEqual(A.data['status'], 'MERGED')
        self.assertEqual(A.reported, 2)

    def test_new_patchset_dequeues_old(self):
        "Test that a new patchset causes the old to be dequeued"
        # D -> C (depends on B) -> B (depends on A) -> A -> M
        self.executor_server.hold_jobs_in_build = True
        M = self.fake_gerrit.addFakeChange('org/project', 'master', 'M')
        M.setMerged()

        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
        C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
        D = self.fake_gerrit.addFakeChange('org/project', 'master', 'D')
        A.addApproval('Code-Review', 2)
        B.addApproval('Code-Review', 2)
        C.addApproval('Code-Review', 2)
        D.addApproval('Code-Review', 2)

        C.setDependsOn(B, 1)
        B.setDependsOn(A, 1)
        A.setDependsOn(M, 1)

        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
        self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
        self.fake_gerrit.addEvent(D.addApproval('Approved', 1))
        self.waitUntilSettled()

        B.addPatchset()
        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(2))
        self.waitUntilSettled()

        self.executor_server.hold_jobs_in_build = False
        self.executor_server.release()
        self.waitUntilSettled()

        self.assertEqual(A.data['status'], 'MERGED')
        self.assertEqual(A.reported, 2)
        self.assertEqual(B.data['status'], 'NEW')
        self.assertEqual(B.reported, 2)
        self.assertEqual(C.data['status'], 'NEW')
        self.assertEqual(C.reported, 2)
        self.assertEqual(D.data['status'], 'MERGED')
        self.assertEqual(D.reported, 2)
        self.assertEqual(len(self.history), 9)  # 3 each for A, B, D.

    @simple_layout('layouts/no-dequeue-on-new-patchset.yaml')
    def test_no_dequeue_on_new_patchset(self):
        "Test the dequeue-on-new-patchset false value"
        A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
        self.executor_server.hold_jobs_in_build = True
        A.addApproval('Code-Review', 2)

        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        self.assertEqual(len(self.builds), 1)

        A.addPatchset()
        A.addApproval('Code-Review', 2)
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(2))
        self.waitUntilSettled()

        self.assertEqual(len(self.builds), 2)

        self.executor_server.hold_jobs_in_build = False
        self.executor_server.release()
        self.waitUntilSettled()

        self.assertHistory([
            dict(name='project1-test', result='SUCCESS', changes='1,1'),
            dict(name='project1-test', result='SUCCESS', changes='1,2'),
        ], ordered=False)

    @simple_layout('layouts/no-dequeue-on-new-patchset.yaml')
    def test_no_dequeue_on_new_patchset_deps(self):
        "Test dependencies are updated if dequeue-on-new-patchset is false"
        A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
        B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
        B.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
            B.subject, A.data['url'])
        A.addApproval('Code-Review', 2)
        B.addApproval('Code-Review', 2)

        self.executor_server.hold_jobs_in_build = True

        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        self.assertEqual(len(self.builds), 1)

        A.addPatchset()
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(2))
        self.waitUntilSettled()

        self.assertEqual(len(self.builds), 0)

        self.executor_server.hold_jobs_in_build = False
        self.executor_server.release()
        self.waitUntilSettled()

        # The item should be dequeued because of the dependency change
        self.assertHistory([
            dict(name='project1-test', result='ABORTED', changes='1,1 2,1'),
        ], ordered=False)

    def test_new_patchset_check(self):
        "Test a new patchset in check"

        self.executor_server.hold_jobs_in_build = True

        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')

        # Add two git-dependent changes
        B.setDependsOn(A, 1)
        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        # A live item, and a non-live/live pair
        items = self.getAllItems('tenant-one', 'check')
        self.assertEqual(len(items), 3)

        self.assertEqual(items[0].changes[0].number, '1')
        self.assertEqual(items[0].changes[0].patchset, '1')
        self.assertFalse(items[0].live)

        self.assertEqual(items[1].changes[0].number, '2')
        self.assertEqual(items[1].changes[0].patchset, '1')
        self.assertTrue(items[1].live)

        self.assertEqual(items[2].changes[0].number, '1')
        self.assertEqual(items[2].changes[0].patchset, '1')
        self.assertTrue(items[2].live)

        # Add a new patchset to A
        A.addPatchset()
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(2))
        self.waitUntilSettled()

        # The live copy of A,1 should be gone, but the non-live and B
        # should continue, and we should have a new A,2
        items = self.getAllItems('tenant-one', 'check')
        self.assertEqual(len(items), 3)

        self.assertEqual(items[0].changes[0].number, '1')
        self.assertEqual(items[0].changes[0].patchset, '1')
        self.assertFalse(items[0].live)

        self.assertEqual(items[1].changes[0].number, '2')
        self.assertEqual(items[1].changes[0].patchset, '1')
        self.assertTrue(items[1].live)

        self.assertEqual(items[2].changes[0].number, '1')
        self.assertEqual(items[2].changes[0].patchset, '2')
        self.assertTrue(items[2].live)

        # Add a new patchset to B
        B.addPatchset()
        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(2))
        self.waitUntilSettled()

        # The live copy of B,1 should be gone, and it's non-live copy of A,1
        # but we should have a new B,2 (still based on A,1)
        items = self.getAllItems('tenant-one', 'check')
        self.assertEqual(len(items), 3)

        self.assertEqual(items[0].changes[0].number, '1')
        self.assertEqual(items[0].changes[0].patchset, '2')
        self.assertTrue(items[0].live)

        self.assertEqual(items[1].changes[0].number, '1')
        self.assertEqual(items[1].changes[0].patchset, '1')
        self.assertFalse(items[1].live)

        self.assertEqual(items[2].changes[0].number, '2')
        self.assertEqual(items[2].changes[0].patchset, '2')
        self.assertTrue(items[2].live)

        self.builds[0].release()
        self.waitUntilSettled()
        self.builds[0].release()
        self.waitUntilSettled()
        self.executor_server.hold_jobs_in_build = False
        self.executor_server.release()
        self.waitUntilSettled()

        self.assertEqual(A.reported, 1)
        self.assertEqual(B.reported, 1)
        self.assertEqual(self.history[0].result, 'ABORTED')
        self.assertEqual(self.history[0].changes, '1,1')
        self.assertEqual(self.history[1].result, 'ABORTED')
        self.assertEqual(self.history[1].changes, '1,1 2,1')
        self.assertEqual(self.history[2].result, 'SUCCESS')
        self.assertEqual(self.history[2].changes, '1,2')
        self.assertEqual(self.history[3].result, 'SUCCESS')
        self.assertEqual(self.history[3].changes, '1,1 2,2')

    def test_abandoned_gate(self):
        "Test that an abandoned change is dequeued from gate"

        self.executor_server.hold_jobs_in_build = True

        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        A.addApproval('Code-Review', 2)
        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.waitUntilSettled()
        self.assertEqual(len(self.builds), 1, "One job being built (on hold)")
        self.assertEqual(self.builds[0].name, 'project-merge')

        self.fake_gerrit.addEvent(A.getChangeAbandonedEvent())
        self.waitUntilSettled()

        self.executor_server.release('.*-merge')
        self.waitUntilSettled()

        self.assertBuilds([])
        self.assertHistory([
            dict(name='project-merge', result='ABORTED', changes='1,1')],
            ordered=False)
        self.assertEqual(A.reported, 1,
                         "Abandoned gate change should report only start")

    def test_abandoned_check(self):
        "Test that an abandoned change is dequeued from check"

        self.executor_server.hold_jobs_in_build = True

        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')

        # Add two git-dependent changes
        B.setDependsOn(A, 1)
        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()
        # A live item, and a non-live/live pair
        items = self.getAllItems('tenant-one', 'check')
        self.assertEqual(len(items), 3)

        self.assertEqual(items[0].changes[0].number, '1')
        self.assertFalse(items[0].live)

        self.assertEqual(items[1].changes[0].number, '2')
        self.assertTrue(items[1].live)

        self.assertEqual(items[2].changes[0].number, '1')
        self.assertTrue(items[2].live)

        # Abandon A
        self.fake_gerrit.addEvent(A.getChangeAbandonedEvent())
        self.waitUntilSettled()

        # The live copy of A should be gone, but the non-live and B
        # should continue
        items = self.getAllItems('tenant-one', 'check')
        self.assertEqual(len(items), 2)

        self.assertEqual(items[0].changes[0].number, '1')
        self.assertFalse(items[0].live)

        self.assertEqual(items[1].changes[0].number, '2')
        self.assertTrue(items[1].live)

        self.executor_server.hold_jobs_in_build = False
        self.executor_server.release()
        self.waitUntilSettled()

        self.assertEqual(len(self.history), 4)
        self.assertEqual(self.history[0].result, 'ABORTED',
                         'Build should have been aborted')
        self.assertEqual(A.reported, 0, "Abandoned change should not report")
        self.assertEqual(B.reported, 1, "Change should report")

    def test_cancel_starting_build(self):
        "Test that a canceled build that is not processed yet is removed"

        self.executor_server.hold_jobs_in_start = True
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        for _ in iterate_timeout(30, 'Wait for build to be in starting phase'):
            if self.executor_server.job_workers:
                break

        tevent = threading.Event()

        def data_watch(event):
            # Set the threading event as soon as the cancel node is present
            tevent.set()

        builds = list(self.executor_api.all())
        # When using DataWatch it is possible for the cancel znode to be
        # created, then deleted almost immediately before the watch handling
        # happens. When this happens kazoo sees no stat info and treats the
        # event as a noop because no new version is available. Use exists to
        # avoid this problem.
        self.zk_client.client.exists(f"{builds[0].path}/cancel", data_watch)

        # Abandon change to cancel build
        self.fake_gerrit.addEvent(A.getChangeAbandonedEvent())

        self.assertTrue(tevent.wait(timeout=30))

        self.executor_server.hold_jobs_in_start = False
        self.waitUntilSettled()

        self.assertHistory([
            dict(name='project-merge', result='ABORTED')
        ])

    def test_abandoned_not_timer(self):
        "Test that an abandoned change does not cancel timer jobs"
        # This test can not use simple_layout because it must start
        # with a configuration which does not include a
        # timer-triggered job so that we have an opportunity to set
        # the hold flag before the first job.
        self.executor_server.hold_jobs_in_build = True
        # Start timer trigger - also org/project
        self.commitConfigUpdate('common-config', 'layouts/idle.yaml')
        self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
        # The pipeline triggers every second, so we should have seen
        # several by now.
        time.sleep(5)
        self.waitUntilSettled()
        # Stop queuing timer triggered jobs so that the assertions
        # below don't race against more jobs being queued.
        # Must be in same repo, so overwrite config with another one
        self.commitConfigUpdate('common-config', 'layouts/no-timer.yaml')
        self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
        self.waitUntilSettled()
        # If APScheduler is in mid-event when we remove the job, we
        # can end up with one more event firing, so give it an extra
        # second to settle.
        time.sleep(1)
        self.waitUntilSettled()

        self.assertEqual(len(self.builds), 1, "One timer job")

        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()
        self.assertEqual(len(self.builds), 2, "One change plus one timer job")

        self.fake_gerrit.addEvent(A.getChangeAbandonedEvent())
        self.waitUntilSettled()

        self.assertEqual(len(self.builds), 1, "One timer job remains")

        self.executor_server.release()
        self.waitUntilSettled()

    def test_timer_branch_updated(self):
        # Test that if a branch in updated while a periodic job is
        # running, we get a second queue item.
        # This test can not use simple_layout because it must start
        # with a configuration which does not include a
        # timer-triggered job so that we have an opportunity to set
        # the hold flag before the first job.
        self.executor_server.hold_jobs_in_build = True
        # Start timer trigger - also org/project
        self.commitConfigUpdate('common-config',
                                'layouts/idle-dereference.yaml')
        self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
        # The pipeline triggers every second, so we should have seen
        # several by now.
        time.sleep(5)
        self.waitUntilSettled()

        M = self.fake_gerrit.addFakeChange('org/project', 'master', 'M')
        M.setMerged()

        time.sleep(5)
        self.waitUntilSettled()

        # Stop queuing timer triggered jobs so that the assertions
        # below don't race against more jobs being queued.
        # Must be in same repo, so overwrite config with another one
        self.commitConfigUpdate('common-config', 'layouts/no-timer.yaml')
        self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
        self.waitUntilSettled()
        # If APScheduler is in mid-event when we remove the job, we
        # can end up with one more event firing, so give it an extra
        # second to settle.
        time.sleep(1)
        self.waitUntilSettled()
        # There should be two periodic jobs, one before and one after
        # the update.
        self.assertEqual(2, len(self.builds))
        self.executor_server.release()
        self.waitUntilSettled()
        self.assertEqual(2, len(self.history))

    def test_new_patchset_dequeues_old_on_head(self):
        "Test that a new patchset causes the old to be dequeued (at head)"
        # D -> C (depends on B) -> B (depends on A) -> A -> M
        self.executor_server.hold_jobs_in_build = True
        M = self.fake_gerrit.addFakeChange('org/project', 'master', 'M')
        M.setMerged()
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
        C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
        D = self.fake_gerrit.addFakeChange('org/project', 'master', 'D')
        A.addApproval('Code-Review', 2)
        B.addApproval('Code-Review', 2)
        C.addApproval('Code-Review', 2)
        D.addApproval('Code-Review', 2)

        C.setDependsOn(B, 1)
        B.setDependsOn(A, 1)
        A.setDependsOn(M, 1)

        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
        self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
        self.fake_gerrit.addEvent(D.addApproval('Approved', 1))
        self.waitUntilSettled()

        A.addPatchset()
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(2))
        self.waitUntilSettled()

        self.executor_server.hold_jobs_in_build = False
        self.executor_server.release()
        self.waitUntilSettled()

        self.assertEqual(A.data['status'], 'NEW')
        self.assertEqual(A.reported, 2)
        self.assertEqual(B.data['status'], 'NEW')
        self.assertEqual(B.reported, 2)
        self.assertEqual(C.data['status'], 'NEW')
        self.assertEqual(C.reported, 2)
        self.assertEqual(D.data['status'], 'MERGED')
        self.assertEqual(D.reported, 2)
        self.assertEqual(len(self.history), 7)

    def test_new_patchset_dequeues_old_without_dependents(self):
        "Test that a new patchset causes only the old to be dequeued"
        self.executor_server.hold_jobs_in_build = True
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
        C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
        A.addApproval('Code-Review', 2)
        B.addApproval('Code-Review', 2)
        C.addApproval('Code-Review', 2)

        self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
        self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.waitUntilSettled()

        B.addPatchset()
        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(2))
        self.waitUntilSettled()

        self.executor_server.hold_jobs_in_build = False
        self.executor_server.release()
        self.waitUntilSettled()

        self.assertEqual(A.data['status'], 'MERGED')
        self.assertEqual(A.reported, 2)
        self.assertEqual(B.data['status'], 'NEW')
        self.assertEqual(B.reported, 2)
        self.assertEqual(C.data['status'], 'MERGED')
        self.assertEqual(C.reported, 2)
        self.assertEqual(len(self.history), 9)

    def test_new_patchset_dequeues_old_independent_queue(self):
        "Test that a new patchset causes the old to be dequeued (independent)"
        self.executor_server.hold_jobs_in_build = True
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
        C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
        self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        B.addPatchset()
        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(2))
        self.waitUntilSettled()

        self.executor_server.hold_jobs_in_build = False
        self.executor_server.release()
        self.waitUntilSettled()

        self.assertEqual(A.data['status'], 'NEW')
        self.assertEqual(A.reported, 1)
        self.assertEqual(B.data['status'], 'NEW')
        self.assertEqual(B.reported, 1)
        self.assertEqual(C.data['status'], 'NEW')
        self.assertEqual(C.reported, 1)
        self.assertEqual(len(self.history), 10)
        self.assertEqual(self.countJobResults(self.history, 'ABORTED'), 1)

    @simple_layout('layouts/noop-job.yaml')
    def test_noop_job(self):
        "Test that the internal noop job works"
        A = self.fake_gerrit.addFakeChange('org/noop-project', 'master', 'A')
        A.addApproval('Code-Review', 2)
        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.waitUntilSettled()

        queue = list(self.executor_api.queued())
        self.assertEqual(len(queue), 0)
        self.assertTrue(self.scheds.first.sched._areAllBuildsComplete())
        self.assertEqual(len(self.history), 0)
        self.assertEqual(A.data['status'], 'MERGED')
        self.assertEqual(A.reported, 2)

    @simple_layout('layouts/no-jobs-project.yaml')
    def test_no_job_project(self):
        "Test that reports with no jobs don't get sent"
        A = self.fake_gerrit.addFakeChange('org/no-jobs-project',
                                           'master', 'A')
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        # Change wasn't reported to
        self.assertEqual(A.reported, False)

        # Check queue is empty afterwards
        items = self.getAllItems('tenant-one', 'check')
        self.assertEqual(len(items), 0)

        self.assertEqual(len(self.history), 0)

    def test_zuul_refs(self):
        "Test that zuul refs exist and have the right changes"
        self.executor_server.hold_jobs_in_build = True
        M1 = self.fake_gerrit.addFakeChange('org/project1', 'master', 'M1')
        M1.setMerged()
        M2 = self.fake_gerrit.addFakeChange('org/project2', 'master', 'M2')
        M2.setMerged()

        A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
        B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
        C = self.fake_gerrit.addFakeChange('org/project2', 'master', 'C')
        D = self.fake_gerrit.addFakeChange('org/project2', 'master', 'D')
        A.addApproval('Code-Review', 2)
        B.addApproval('Code-Review', 2)
        C.addApproval('Code-Review', 2)
        D.addApproval('Code-Review', 2)
        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
        self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
        self.fake_gerrit.addEvent(D.addApproval('Approved', 1))

        self.waitUntilSettled()
        self.executor_server.release('.*-merge')
        self.waitUntilSettled()
        self.executor_server.release('.*-merge')
        self.waitUntilSettled()
        self.executor_server.release('.*-merge')
        self.waitUntilSettled()
        self.executor_server.release('.*-merge')
        self.waitUntilSettled()

        a_build = b_build = c_build = d_build = None
        for x in self.builds:
            if x.parameters['zuul']['change'] == '3':
                a_build = x
            elif x.parameters['zuul']['change'] == '4':
                b_build = x
            elif x.parameters['zuul']['change'] == '5':
                c_build = x
            elif x.parameters['zuul']['change'] == '6':
                d_build = x
            if a_build and b_build and c_build and d_build:
                break

        # should have a, not b, and should not be in project2
        self.assertTrue(a_build.hasChanges(A))
        self.assertFalse(a_build.hasChanges(B, M2))

        # should have a and b, and should not be in project2
        self.assertTrue(b_build.hasChanges(A, B))
        self.assertFalse(b_build.hasChanges(M2))

        # should have a and b in 1, c in 2
        self.assertTrue(c_build.hasChanges(A, B, C))
        self.assertFalse(c_build.hasChanges(D))

        # should have a and b in 1, c and d in 2
        self.assertTrue(d_build.hasChanges(A, B, C, D))

        self.executor_server.hold_jobs_in_build = False
        self.executor_server.release()
        self.waitUntilSettled()

        self.assertEqual(A.data['status'], 'MERGED')
        self.assertEqual(A.reported, 2)
        self.assertEqual(B.data['status'], 'MERGED')
        self.assertEqual(B.reported, 2)
        self.assertEqual(C.data['status'], 'MERGED')
        self.assertEqual(C.reported, 2)
        self.assertEqual(D.data['status'], 'MERGED')
        self.assertEqual(D.reported, 2)

    def test_rerun_on_error(self):
        "Test that if a worker fails to run a job, it is run again"
        self.executor_server.hold_jobs_in_build = True
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        A.addApproval('Code-Review', 2)
        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.waitUntilSettled()

        self.builds[0].requeue = True
        self.executor_server.hold_jobs_in_build = False
        self.executor_server.release()
        self.waitUntilSettled()
        self.assertEqual(self.countJobResults(self.history, None), 1)
        self.assertEqual(self.countJobResults(self.history, 'SUCCESS'), 3)

    def test_statsd(self):
        "Test each of the statsd methods used in the scheduler"
        statsd = self.scheds.first.sched.statsd
        statsd.incr('test-incr')
        statsd.timing('test-timing', 3)
        statsd.gauge('test-gauge', 12)
        self.assertReportedStat('test-incr', '1', 'c')
        self.assertReportedStat('test-timing', '3', 'ms')
        self.assertReportedStat('test-gauge', '12', 'g')

        # test key normalization
        statsd.extra_keys = {'hostname': '1_2_3_4'}

        statsd.incr('hostname-incr.{hostname}.{fake}', fake='1:2')
        statsd.timing('hostname-timing.{hostname}.{fake}', 3, fake='1:2')
        statsd.gauge('hostname-gauge.{hostname}.{fake}', 12, fake='1:2')
        self.assertReportedStat('hostname-incr.1_2_3_4.1_2', '1', 'c')
        self.assertReportedStat('hostname-timing.1_2_3_4.1_2', '3', 'ms')
        self.assertReportedStat('hostname-gauge.1_2_3_4.1_2', '12', 'g')

    def test_statsd_conflict(self):
        statsd = self.scheds.first.sched.statsd
        statsd.gauge('test-gauge', 12)
        # since test-gauge is already a value, we can't make
        # subvalues.  Test the assert works.
        statsd.gauge('test-gauge.1_2_3_4', 12)
        self.assertReportedStat('test-gauge', '12', 'g')
        self.assertRaises(Exception, self.assertReportedStat,
                          'test-gauge.1_2_3_4', '12', 'g')

    def test_file_head(self):
        # This is a regression test for an observed bug.  A change
        # with a file named "HEAD" in the root directory of the repo
        # was processed by a merger.  It then was unable to reset the
        # repo because of:
        #   GitCommandError: 'git reset --hard HEAD' returned
        #       with exit code 128
        #   stderr: 'fatal: ambiguous argument 'HEAD': both revision
        #       and filename
        #   Use '--' to separate filenames from revisions'

        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        A.addPatchset({'HEAD': ''})
        B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')

        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(2))
        self.waitUntilSettled()

        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        self.assertIn('Build succeeded', A.messages[0])
        self.assertIn('Build succeeded', B.messages[0])

    def test_file_jobs(self):
        "Test that file jobs run only when appropriate"
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        A.addPatchset({'pip-requires': 'foo'})
        B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
        A.addApproval('Code-Review', 2)
        B.addApproval('Code-Review', 2)
        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
        self.waitUntilSettled()

        testfile_jobs = [x for x in self.history
                         if x.name == 'project-testfile']

        self.assertEqual(len(testfile_jobs), 1)
        self.assertEqual(testfile_jobs[0].changes, '1,2')
        self.assertEqual(A.data['status'], 'MERGED')
        self.assertEqual(A.reported, 2)
        self.assertEqual(B.data['status'], 'MERGED')
        self.assertEqual(B.reported, 2)

    def _test_files_negated_jobs(self, should_skip):
        "Test that jobs with negated files filter run only when appropriate"
        if should_skip:
            files = {'dontrun': 'me\n'}
        else:
            files = {'dorun': 'please!\n'}

        change = self.fake_gerrit.addFakeChange('org/project',
                                                'master',
                                                'test files',
                                                files=files)
        self.fake_gerrit.addEvent(change.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        tested_change_ids = [x.changes[0] for x in self.history
                             if x.name == 'project-test-files']

        if should_skip:
            self.assertEqual([], tested_change_ids)
        else:
            self.assertIn(change.data['number'], tested_change_ids)

    @simple_layout('layouts/files-negate.yaml')
    def test_files_negated_no_match_skips_job(self):
        self._test_files_negated_jobs(should_skip=True)

    @simple_layout('layouts/files-negate.yaml')
    def test_files_negated_match_runs_job(self):
        self._test_files_negated_jobs(should_skip=False)

    def _test_irrelevant_files_jobs(self, should_skip):
        "Test that jobs with irrelevant-files filter run only when appropriate"
        if should_skip:
            files = {'ignoreme': 'ignored\n'}
        else:
            files = {'respectme': 'please!\n'}

        change = self.fake_gerrit.addFakeChange('org/project',
                                                'master',
                                                'test irrelevant-files',
                                                files=files)
        self.fake_gerrit.addEvent(change.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        tested_change_ids = [x.changes[0] for x in self.history
                             if x.name == 'project-test-irrelevant-files']

        if should_skip:
            self.assertEqual([], tested_change_ids)
        else:
            self.assertIn(change.data['number'], tested_change_ids)

    @simple_layout('layouts/irrelevant-files.yaml')
    def test_irrelevant_files_match_skips_job(self):
        self._test_irrelevant_files_jobs(should_skip=True)

    @simple_layout('layouts/irrelevant-files.yaml')
    def test_irrelevant_files_no_match_runs_job(self):
        self._test_irrelevant_files_jobs(should_skip=False)

    def _test_irrelevant_files_negated_jobs(self, should_skip):
        "Test that jobs with irrelevant-files filter run only when appropriate"
        if should_skip:
            files = {'ignoreme': 'ignored\n'}
        else:
            files = {'respectme': 'please!\n'}

        change = self.fake_gerrit.addFakeChange('org/project',
                                                'master',
                                                'test irrelevant-files',
                                                files=files)
        self.fake_gerrit.addEvent(change.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        tested_change_ids = [x.changes[0] for x in self.history
                             if x.name == 'project-test-irrelevant-files']

        if should_skip:
            self.assertEqual([], tested_change_ids)
        else:
            self.assertIn(change.data['number'], tested_change_ids)

    @simple_layout('layouts/irrelevant-files-negate.yaml')
    def test_irrelevant_files_negated_match_skips_job(self):
        # Anything other than "respectme" is irrelevant. This adds
        # "README" which is irrelevant, and "ignoreme" which is
        # irrelevant, so the job should not run.
        self._test_irrelevant_files_negated_jobs(should_skip=True)

    @simple_layout('layouts/irrelevant-files-negate.yaml')
    def test_irrelevant_files_negated_no_match_runs_job(self):
        # Anything other than "respectme" is irrelevant. This adds
        # "README" which is irrelevant, and "respecme" which *is*
        # relevant, so the job should run.
        self._test_irrelevant_files_negated_jobs(should_skip=False)

    @simple_layout('layouts/inheritance.yaml')
    def test_inherited_jobs_keep_matchers(self):
        files = {'ignoreme': 'ignored\n'}

        change = self.fake_gerrit.addFakeChange('org/project',
                                                'master',
                                                'test irrelevant-files',
                                                files=files)
        self.fake_gerrit.addEvent(change.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        run_jobs = set([build.name for build in self.history])

        self.assertEqual(set(['project-test-nomatch-starts-empty',
                              'project-test-nomatch-starts-full']), run_jobs)

    @simple_layout('layouts/job-vars.yaml')
    def test_inherited_job_variables(self):
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()
        self.assertHistory([
            dict(name='parentjob', result='SUCCESS'),
            dict(name='child1', result='SUCCESS'),
            dict(name='child2', result='SUCCESS'),
            dict(name='child3', result='SUCCESS'),
            dict(name='override_project_var', result='SUCCESS'),
            dict(name='job_from_template1', result='SUCCESS'),
            dict(name='job_from_template2', result='SUCCESS'),
        ], ordered=False)
        j = self.getJobFromHistory('parentjob')
        rp = set([p['name'] for p in j.parameters['projects']])
        job_vars = j.job.variables
        self.assertEqual(job_vars['project_var'], 'set_in_project')
        self.assertEqual(job_vars['template_var1'], 'set_in_template1')
        self.assertEqual(job_vars['template_var2'], 'set_in_template2')
        self.assertEqual(job_vars['override'], 0)
        self.assertEqual(job_vars['child1override'], 0)
        self.assertEqual(job_vars['parent'], 0)
        self.assertEqual(job_vars['deep']['override'], 0)
        self.assertFalse('child1' in job_vars)
        self.assertFalse('child2' in job_vars)
        self.assertFalse('child3' in job_vars)
        self.assertEqual(rp, set(['org/project', 'org/project0',
                                  'org/project0']))
        j = self.getJobFromHistory('child1')
        rp = set([p['name'] for p in j.parameters['projects']])
        job_vars = j.job.variables
        self.assertEqual(job_vars['project_var'], 'set_in_project')
        self.assertEqual(job_vars['override'], 1)
        self.assertEqual(job_vars['child1override'], 1)
        self.assertEqual(job_vars['parent'], 0)
        self.assertEqual(job_vars['child1'], 1)
        self.assertEqual(job_vars['deep']['override'], 1)
        self.assertFalse('child2' in job_vars)
        self.assertFalse('child3' in job_vars)
        self.assertEqual(rp, set(['org/project', 'org/project0',
                                  'org/project1']))
        j = self.getJobFromHistory('child2')
        job_vars = j.job.variables
        self.assertEqual(job_vars['project_var'], 'set_in_project')
        rp = set([p['name'] for p in j.parameters['projects']])
        self.assertEqual(job_vars['override'], 2)
        self.assertEqual(job_vars['child1override'], 0)
        self.assertEqual(job_vars['parent'], 0)
        self.assertEqual(job_vars['deep']['override'], 2)
        self.assertFalse('child1' in job_vars)
        self.assertEqual(job_vars['child2'], 2)
        self.assertFalse('child3' in job_vars)
        self.assertEqual(rp, set(['org/project', 'org/project0',
                                  'org/project2']))
        j = self.getJobFromHistory('child3')
        job_vars = j.job.variables
        self.assertEqual(job_vars['project_var'], 'set_in_project')
        rp = set([p['name'] for p in j.parameters['projects']])
        self.assertEqual(job_vars['override'], 3)
        self.assertEqual(job_vars['child1override'], 0)
        self.assertEqual(job_vars['parent'], 0)
        self.assertEqual(job_vars['deep']['override'], 3)
        self.assertFalse('child1' in job_vars)
        self.assertFalse('child2' in job_vars)
        self.assertEqual(job_vars['child3'], 3)
        self.assertEqual(rp, set(['org/project', 'org/project0',
                                  'org/project3']))
        j = self.getJobFromHistory('override_project_var')
        job_vars = j.job.variables
        self.assertEqual(job_vars['project_var'], 'override_in_job')

    @simple_layout('layouts/job-variants.yaml')
    def test_job_branch_variants(self):
        self.create_branch('org/project', 'stable/diablo')
        self.fake_gerrit.addEvent(
            self.fake_gerrit.getFakeBranchCreatedEvent(
                'org/project', 'stable/diablo'))
        self.create_branch('org/project', 'stable/essex')
        self.fake_gerrit.addEvent(
            self.fake_gerrit.getFakeBranchCreatedEvent(
                'org/project', 'stable/essex'))
        self.waitUntilSettled()

        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        B = self.fake_gerrit.addFakeChange('org/project', 'stable/diablo', 'B')
        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        C = self.fake_gerrit.addFakeChange('org/project', 'stable/essex', 'C')
        self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()
        self.assertHistory([
            dict(name='python27', result='SUCCESS'),
            dict(name='python27', result='SUCCESS'),
            dict(name='python27', result='SUCCESS'),
        ])

        j = self.history[0].job
        self.assertEqual(j.timeout, 40)
        self.assertEqual(len(j.nodeset.nodes), 1)
        self.assertEqual(next(iter(j.nodeset.nodes.values())).label, 'new')
        self.assertEqual([x['path'] for x in j.pre_run],
                         ['base-pre', 'py27-pre'])
        self.assertEqual([x['path'] for x in j.post_run],
                         ['py27-post-a', 'py27-post-b', 'base-post'])
        self.assertEqual([x['path'] for x in j.run],
                         ['playbooks/python27.yaml'])

        j = self.history[1].job
        self.assertEqual(j.timeout, 50)
        self.assertEqual(len(j.nodeset.nodes), 1)
        self.assertEqual(next(iter(j.nodeset.nodes.values())).label, 'old')
        self.assertEqual([x['path'] for x in j.pre_run],
                         ['base-pre', 'py27-pre', 'py27-diablo-pre'])
        self.assertEqual([x['path'] for x in j.post_run],
                         ['py27-diablo-post', 'py27-post-a', 'py27-post-b',
                          'base-post'])
        self.assertEqual([x['path'] for x in j.run],
                         ['py27-diablo'])

        j = self.history[2].job
        self.assertEqual(j.timeout, 40)
        self.assertEqual(len(j.nodeset.nodes), 1)
        self.assertEqual(next(iter(j.nodeset.nodes.values())).label, 'new')
        self.assertEqual([x['path'] for x in j.pre_run],
                         ['base-pre', 'py27-pre', 'py27-essex-pre'])
        self.assertEqual([x['path'] for x in j.post_run],
                         ['py27-essex-post', 'py27-post-a', 'py27-post-b',
                          'base-post'])
        self.assertEqual([x['path'] for x in j.run],
                         ['playbooks/python27.yaml'])

    @simple_layout("layouts/no-run.yaml")
    def test_job_without_run(self):
        "Test that a job without a run playbook errors"
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()
        self.assertIn('Job base does not specify a run playbook',
                      A.messages[-1])

    def test_queue_names(self):
        "Test shared change queue names"
        tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
        (trusted, project1) = tenant.getProject('org/project1')
        (trusted, project2) = tenant.getProject('org/project2')
        # Change queues are created lazy by the dependent pipeline manager
        # so retrieve the queue first without having to really enqueue a
        # change first.
        gate = tenant.layout.pipeline_managers['gate']
        FakeChange = namedtuple('FakeChange', ['project', 'branch'])
        fake_a = FakeChange(project1, 'master')
        fake_b = FakeChange(project2, 'master')
        with (pipeline_lock(self.zk_client, tenant.name,
                            gate.pipeline.name) as lock,
              self.createZKContext(lock) as ctx,
              gate.currentContext(ctx)):
            gate.change_list.refresh(ctx)
            gate.state.refresh(ctx)
            gate.getChangeQueue(fake_a, None)
            gate.getChangeQueue(fake_b, None)
        q1 = gate.state.getQueue(project1.canonical_name, None)
        q2 = gate.state.getQueue(project2.canonical_name, None)
        self.assertEqual(q1.name, 'integrated')
        self.assertEqual(q2.name, 'integrated')

    @simple_layout("layouts/template-queue.yaml")
    def test_template_queue(self):
        "Test a shared queue can be constructed from a project-template"
        tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
        (trusted, project1) = tenant.getProject('org/project1')
        (trusted, project2) = tenant.getProject('org/project2')

        # Change queues are created lazy by the dependent pipeline manager
        # so retrieve the queue first without having to really enqueue a
        # change first.
        gate = tenant.layout.pipeline_managers['gate']
        FakeChange = namedtuple('FakeChange', ['project', 'branch'])
        fake_a = FakeChange(project1, 'master')
        fake_b = FakeChange(project2, 'master')
        with (pipeline_lock(self.zk_client, tenant.name,
                            gate.pipeline.name) as lock,
              self.createZKContext(lock) as ctx,
              gate.currentContext(ctx)):
            gate.change_list.refresh(ctx)
            gate.state.refresh(ctx)
            gate.getChangeQueue(fake_a, None)
            gate.getChangeQueue(fake_b, None)
        q1 = gate.state.getQueue(project1.canonical_name, None)
        q2 = gate.state.getQueue(project2.canonical_name, None)
        self.assertEqual(q1.name, 'integrated')
        self.assertEqual(q2.name, 'integrated')

    @simple_layout("layouts/template-project-queue.yaml")
    def test_template_project_queue(self):
        "Test a shared queue can be constructed from a project-template"
        tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
        (trusted, project1) = tenant.getProject('org/project1')
        (trusted, project2) = tenant.getProject('org/project2')

        # Change queues are created lazy by the dependent pipeline manager
        # so retrieve the queue first without having to really enqueue a
        # change first.
        gate = tenant.layout.pipeline_managers['gate']
        FakeChange = namedtuple('FakeChange', ['project', 'branch'])
        fake_a = FakeChange(project1, 'master')
        fake_b = FakeChange(project2, 'master')
        with (pipeline_lock(self.zk_client, tenant.name,
                            gate.pipeline.name) as lock,
              self.createZKContext(lock) as ctx,
              gate.currentContext(ctx)):
            gate.change_list.refresh(ctx)
            gate.state.refresh(ctx)
            gate.getChangeQueue(fake_a, None)
            gate.getChangeQueue(fake_b, None)
        q1 = gate.state.getQueue(project1.canonical_name, None)
        q2 = gate.state.getQueue(project2.canonical_name, None)
        self.assertEqual(q1.name, 'integrated')
        self.assertEqual(q2.name, 'integrated')

    @simple_layout("layouts/regex-template-queue.yaml")
    def test_regex_template_queue(self):
        "Test a shared queue can be constructed from a regex project-template"
        tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
        (trusted, project1) = tenant.getProject('org/project1')
        (trusted, project2) = tenant.getProject('org/project2')
        # Change queues are created lazy by the dependent pipeline manager
        # so retrieve the queue first without having to really enqueue a
        # change first.
        gate = tenant.layout.pipeline_managers['gate']
        FakeChange = namedtuple('FakeChange', ['project', 'branch'])
        fake_a = FakeChange(project1, 'master')
        fake_b = FakeChange(project2, 'master')
        with (pipeline_lock(self.zk_client, tenant.name,
                            gate.pipeline.name) as lock,
              self.createZKContext(lock) as ctx,
              gate.currentContext(ctx)):
            gate.change_list.refresh(ctx)
            gate.state.refresh(ctx)
            gate.getChangeQueue(fake_a, None)
            gate.getChangeQueue(fake_b, None)
        q1 = gate.state.getQueue(project1.canonical_name, None)
        q2 = gate.state.getQueue(project2.canonical_name, None)
        self.assertEqual(q1.name, 'integrated')
        self.assertEqual(q2.name, 'integrated')

    @simple_layout("layouts/regex-queue.yaml")
    @skipIfMultiScheduler()
    def test_regex_queue(self):
        "Test a shared queue can be constructed from a regex project"
        tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
        (trusted, project1) = tenant.getProject('org/project1')
        (trusted, project2) = tenant.getProject('org/project2')
        # Change queues are created lazy by the dependent pipeline manager
        # so retrieve the queue first without having to really enqueue a
        # change first.
        gate = tenant.layout.pipeline_managers['gate']
        FakeChange = namedtuple('FakeChange', ['project', 'branch'])
        fake_a = FakeChange(project1, 'master')
        fake_b = FakeChange(project2, 'master')
        with (pipeline_lock(self.zk_client, tenant.name,
                            gate.pipeline.name) as lock,
              self.createZKContext(lock) as ctx,
              gate.currentContext(ctx)):
            gate.change_list.refresh(ctx)
            gate.state.refresh(ctx)
            gate.getChangeQueue(fake_a, None)
            gate.getChangeQueue(fake_b, None)
        q1 = gate.state.getQueue(project1.canonical_name, None)
        q2 = gate.state.getQueue(project2.canonical_name, None)
        self.assertEqual(q1.name, 'integrated')
        self.assertEqual(q2.name, 'integrated')

    def test_queue_precedence(self):
        "Test that queue precedence works"

        self.hold_jobs_in_queue = True
        self.executor_server.hold_jobs_in_build = True
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        A.addApproval('Code-Review', 2)
        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

        self.waitUntilSettled()
        self.hold_jobs_in_queue = False
        self.executor_api.release()
        self.waitUntilSettled()

        # Run one build at a time to ensure non-race order:
        self.orderedRelease()
        self.executor_server.hold_jobs_in_build = False
        self.waitUntilSettled()

        self.log.debug(self.history)
        self.assertEqual(self.history[0].pipeline, 'gate')
        self.assertEqual(self.history[1].pipeline, 'check')
        self.assertEqual(self.history[2].pipeline, 'gate')
        self.assertEqual(self.history[3].pipeline, 'gate')
        self.assertEqual(self.history[4].pipeline, 'check')
        self.assertEqual(self.history[5].pipeline, 'check')

    @simple_layout('layouts/two-check.yaml')
    def test_query_dependency_count(self):
        # Test that we efficiently query dependent changes
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
        B.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
            B.subject, A.data['url'])
        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()
        # 1. The query to find the change id
        #      from the Depends-On string "change:1" (simpleQuery)
        # 2. The query to populate the change once we know the id
        #      (queryChange)
        self.assertEqual(A.queried, 2)
        self.assertEqual(B.queried, 1)

    def test_reconfigure_merge(self):
        """Test that two reconfigure events are merged"""
        # Wrap the recofiguration handler so we can count how many
        # times it runs.
        with mock.patch.object(
                zuul.scheduler.Scheduler, '_doTenantReconfigureEvent',
                wraps=self.scheds.first.sched._doTenantReconfigureEvent
        ) as mymock:
            with self.scheds.first.sched.run_handler_lock:
                self.create_branch('org/project', 'stable/diablo')
                self.fake_gerrit.addEvent(
                    self.fake_gerrit.getFakeBranchCreatedEvent(
                        'org/project', 'stable/diablo'))
                self.create_branch('org/project', 'stable/essex')
                self.fake_gerrit.addEvent(
                    self.fake_gerrit.getFakeBranchCreatedEvent(
                        'org/project', 'stable/essex'))
                for _ in iterate_timeout(60, 'jobs started'):
                    if len(self.scheds.first.sched.trigger_events[
                            'tenant-one']) == 2:
                        break

            self.waitUntilSettled()
            mymock.assert_called_once()

    def test_live_reconfiguration(self):
        "Test that live reconfiguration works"
        self.executor_server.hold_jobs_in_build = True
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        A.addApproval('Code-Review', 2)
        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.waitUntilSettled()

        self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
        self.waitUntilSettled()

        self.executor_server.hold_jobs_in_build = False
        self.executor_server.release()
        self.waitUntilSettled()
        self.assertEqual(self.getJobFromHistory('project-merge').result,
                         'SUCCESS')
        self.assertEqual(self.getJobFromHistory('project-test1').result,
                         'SUCCESS')
        self.assertEqual(self.getJobFromHistory('project-test2').result,
                         'SUCCESS')
        self.assertEqual(A.data['status'], 'MERGED')
        self.assertEqual(A.reported, 2)
        events = self.scheds.first.sched.sql.connection.getSystemEvents()
        self.assertEqual(1, len(events))
        self.assertEqual("tenant reconfiguration", events[0].event_type)
        self.assertEqual("Tenant reconfiguration due to external request",
                         events[0].description)

    def test_live_reconfiguration_layout_cache_fallback(self):
        # Test that re-calculating a dynamic fallback layout works after it
        # was removed during a reconfiguration.
        self.executor_server.hold_jobs_in_build = True

        in_repo_conf = textwrap.dedent(
            """
            - job:
                name: project-test3
                parent: project-test1

            # add a job by the canonical project name
            - project:
                gate:
                  jobs:
                    - project-test3:
                        dependencies:
                          - project-merge
            """)

        file_dict = {'zuul.d/a.yaml': in_repo_conf}

        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
                                           files=file_dict)
        A.addApproval('Code-Review', 2)
        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.waitUntilSettled()

        self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
        self.waitUntilSettled()

        items = self.getAllItems('tenant-one', 'gate')
        self.assertEqual(len(items), 1)
        self.assertIsNone(items[0].layout_uuid)

        # Assert that the layout cache is empty after a reconfiguration.
        tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
        manager = tenant.layout.pipeline_managers['gate']
        self.assertEqual(manager._layout_cache, {})

        B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B',
                                           parent='refs/changes/01/1/1')
        B.addApproval('Code-Review', 2)

        self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
        self.waitUntilSettled()

        items = self.getAllItems('tenant-one', 'gate')
        self.assertEqual(len(items), 2)
        for item in items:
            # Layout UUID should be set again for all live items. It had to
            # be re-calculated for the first item in the queue as it was reset
            # during re-enqueue.
            self.assertIsNotNone(item.layout_uuid)

        self.executor_server.hold_jobs_in_build = False
        self.executor_server.release()
        self.waitUntilSettled()

        self.assertEqual(A.data['status'], 'MERGED')
        self.assertEqual(A.reported, 2)
        self.assertEqual(B.data['status'], 'MERGED')
        self.assertEqual(B.reported, 2)

        self.assertHistory([
            dict(name='project-merge', result='SUCCESS', changes='1,1'),
            dict(name='project-merge', result='SUCCESS', changes='1,1 2,1'),
            dict(name='project-test1', result='SUCCESS', changes='1,1'),
            dict(name='project-test1', result='SUCCESS', changes='1,1 2,1'),
            dict(name='project-test2', result='SUCCESS', changes='1,1'),
            dict(name='project-test2', result='SUCCESS', changes='1,1 2,1'),
            dict(name='project-test3', result='SUCCESS', changes='1,1'),
            dict(name='project-test3', result='SUCCESS', changes='1,1 2,1'),
        ], ordered=False)

    def test_live_reconfiguration_layout_cache_non_live(self):
        # Test that the layout UUID is only reset for live items.
        self.executor_server.hold_jobs_in_build = True

        in_repo_conf = textwrap.dedent(
            """
            - job:
                name: project-test3
                parent: project-test1

            # add a job by the canonical project name
            - project:
                check:
                  jobs:
                    - project-test3:
                        dependencies:
                          - project-merge
            """)

        file_dict = {'zuul.d/a.yaml': in_repo_conf}

        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
                                           files=file_dict)
        B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B',
                                           parent='refs/changes/01/1/1')
        B.setDependsOn(A, 1)
        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
        self.waitUntilSettled()

        items = self.getAllItems('tenant-one', 'check')
        self.assertEqual(len(items), 2)

        # Assert that the layout UUID of the live item is reset during a
        # reconfiguration, but non-live items keep their UUID.
        self.assertIsNotNone(items[0].layout_uuid)
        self.assertIsNone(items[1].layout_uuid)

        # Cache should be empty after a reconfiguration
        tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
        manager = tenant.layout.pipeline_managers['check']
        self.assertEqual(manager._layout_cache, {})

        self.executor_server.hold_jobs_in_build = False
        self.executor_server.release()
        self.waitUntilSettled()

        self.assertEqual(A.data['status'], 'NEW')
        self.assertEqual(A.reported, 0)
        self.assertEqual(B.data['status'], 'NEW')
        self.assertEqual(B.reported, 1)

        self.assertHistory([
            dict(name='project-merge', result='SUCCESS', changes='1,1 2,1'),
            dict(name='project-test1', result='SUCCESS', changes='1,1 2,1'),
            dict(name='project-test2', result='SUCCESS', changes='1,1 2,1'),
            dict(name='project-test3', result='SUCCESS', changes='1,1 2,1'),
        ], ordered=False)

    def test_live_reconfiguration_command_socket(self):
        "Test that live reconfiguration via command socket works"

        # record previous tenant reconfiguration state, which may not be set
        old = self.scheds.first.sched.tenant_layout_state.get(
            'tenant-one', EMPTY_LAYOUT_STATE)
        self.waitUntilSettled()

        # Layout last reconfigured time resolution is 1 second
        time.sleep(1)
        command_socket = self.scheds.first.config.get(
            'scheduler', 'command_socket')
        with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as s:
            s.connect(command_socket)
            s.sendall('full-reconfigure\n'.encode('utf8'))

        # Wait for full reconfiguration. Note that waitUntilSettled is not
        # reliable here because the reconfigure event may arrive in the
        # event queue after waitUntilSettled.
        start = time.time()
        while True:
            if time.time() - start > 15:
                raise Exception("Timeout waiting for full reconfiguration")
            new = self.scheds.first.sched.tenant_layout_state.get(
                'tenant-one', EMPTY_LAYOUT_STATE)
            if old < new:
                break
            else:
                time.sleep(0)

        self.assertGreater(new.last_reconfigured, old.last_reconfigured)
        self.assertGreater(new.last_reconfigure_event_ltime,
                           old.last_reconfigure_event_ltime)

    def test_tenant_reconfiguration_command_socket(self):
        "Test that single-tenant reconfiguration via command socket works"

        # record previous tenant reconfiguration state, which may not be set
        old = self.scheds.first.sched.tenant_layout_state.get(
            'tenant-one', EMPTY_LAYOUT_STATE)
        self.waitUntilSettled()

        command_socket = self.scheds.first.config.get(
            'scheduler', 'command_socket')
        with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as s:
            s.connect(command_socket)
            s.sendall('tenant-reconfigure ["tenant-one"]\n'.encode('utf8'))

        # Wait for full reconfiguration. Note that waitUntilSettled is not
        # reliable here because the reconfigure event may arrive in the
        # event queue after waitUntilSettled.
        start = time.time()
        while True:
            if time.time() - start > 15:
                raise Exception("Timeout waiting for full reconfiguration")
            new = self.scheds.first.sched.tenant_layout_state.get(
                'tenant-one', EMPTY_LAYOUT_STATE)
            if old < new:
                break
            else:
                time.sleep(0)

    def test_double_live_reconfiguration_shared_queue(self):
        # This was a real-world regression.  A change is added to
        # gate; a reconfigure happens, a second change which depends
        # on the first is added, and a second reconfiguration happens.
        # Ensure that both changes merge.

        # A failure may indicate incorrect caching or cleaning up of
        # references during a reconfiguration.
        self.executor_server.hold_jobs_in_build = True

        A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
        B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
        B.setDependsOn(A, 1)
        A.addApproval('Code-Review', 2)
        B.addApproval('Code-Review', 2)

        # Add the parent change.
        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.waitUntilSettled()
        self.executor_server.release('.*-merge')
        self.waitUntilSettled()

        # Reconfigure (with only one change in the pipeline).
        self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
        self.waitUntilSettled()

        # Add the child change.
        self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
        self.waitUntilSettled()
        self.executor_server.release('.*-merge')
        self.waitUntilSettled()

        # Reconfigure (with both in the pipeline).
        self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
        self.waitUntilSettled()

        self.executor_server.hold_jobs_in_build = False
        self.executor_server.release()
        self.waitUntilSettled()

        self.assertEqual(len(self.history), 8)

        self.assertEqual(A.data['status'], 'MERGED')
        self.assertEqual(A.reported, 2)
        self.assertEqual(B.data['status'], 'MERGED')
        self.assertEqual(B.reported, 2)

    def test_live_reconfiguration_del_project(self):
        # Test project deletion from tenant while changes are enqueued

        self.executor_server.hold_jobs_in_build = True
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
        C = self.fake_gerrit.addFakeChange('org/project1', 'master', 'C')

        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()
        self.executor_server.release('.*-merge')
        self.waitUntilSettled()
        self.assertEqual(len(self.builds), 8)

        self.newTenantConfig('config/single-tenant/main-one-project.yaml')
        # This layout defines only org/project, not org/project1
        self.commitConfigUpdate(
            'common-config',
            'layouts/live-reconfiguration-del-project.yaml')
        self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
        self.waitUntilSettled()

        self.executor_server.hold_jobs_in_build = False
        self.executor_server.release()
        self.waitUntilSettled()

        self.assertEqual(A.data['status'], 'NEW')
        self.assertEqual(B.data['status'], 'NEW')
        self.assertEqual(C.data['status'], 'NEW')
        self.assertEqual(A.reported, 1)
        self.assertEqual(B.reported, 0)
        self.assertEqual(C.reported, 0)

        self.assertHistory([
            dict(name='project-merge', result='SUCCESS', changes='2,1'),
            dict(name='project-merge', result='SUCCESS', changes='1,1'),
            dict(name='project-merge', result='SUCCESS', changes='3,1'),
            dict(name='project-test1', result='ABORTED', changes='2,1'),
            dict(name='project-test2', result='ABORTED', changes='2,1'),
            dict(name='project1-project2-integration',
                 result='ABORTED', changes='2,1'),
            dict(name='project-test1', result='ABORTED', changes='3,1'),
            dict(name='project-test2', result='ABORTED', changes='3,1'),
            dict(name='project1-project2-integration',
                 result='ABORTED', changes='3,1'),
            dict(name='project-test1', result='SUCCESS', changes='1,1'),
            dict(name='project-test2', result='SUCCESS', changes='1,1'),
        ], ordered=False)

        tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
        self.assertEqual(
            len(tenant.layout.pipeline_managers['check'].state.queues), 0)
        self.assertIn('Build succeeded', A.messages[0])

    def test_live_reconfiguration_del_pipeline(self):
        # Test pipeline deletion while changes are enqueued

        self.executor_server.hold_jobs_in_build = True
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()
        self.executor_server.release('.*-merge')
        self.waitUntilSettled()
        self.assertEqual(len(self.builds), 2)

        # This layout defines only org/project, not org/project1
        self.commitConfigUpdate(
            'common-config',
            'layouts/live-reconfiguration-del-pipeline.yaml')
        self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
        self.waitUntilSettled()

        self.executor_server.hold_jobs_in_build = False
        self.executor_server.release()
        self.waitUntilSettled()

        self.assertEqual(A.data['status'], 'NEW')
        self.assertEqual(A.reported, 0)

        self.assertHistory([
            dict(name='project-merge', result='SUCCESS', changes='1,1'),
            dict(name='project-test1', result='ABORTED', changes='1,1'),
            dict(name='project-test2', result='ABORTED', changes='1,1'),
        ], ordered=False)

        tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
        self.assertEqual(len(tenant.layout.pipeline_managers), 0)

    def test_live_reconfiguration_del_tenant(self):
        # Test tenant deletion while changes are enqueued

        self.executor_server.hold_jobs_in_build = True
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
        C = self.fake_gerrit.addFakeChange('org/project1', 'master', 'C')

        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()
        self.executor_server.release('.*-merge')
        self.waitUntilSettled()
        self.assertEqual(len(self.builds), 8)

        self.newTenantConfig('config/single-tenant/main-no-tenants.yaml')
        self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
        self.waitUntilSettled()

        self.executor_server.hold_jobs_in_build = False
        self.executor_server.release()
        self.waitUntilSettled()

        self.assertEqual(A.data['status'], 'NEW')
        self.assertEqual(B.data['status'], 'NEW')
        self.assertEqual(C.data['status'], 'NEW')
        self.assertEqual(A.reported, 0)
        self.assertEqual(B.reported, 0)
        self.assertEqual(C.reported, 0)

        self.assertHistory([
            dict(name='project-merge', result='SUCCESS', changes='2,1'),
            dict(name='project-merge', result='SUCCESS', changes='1,1'),
            dict(name='project-merge', result='SUCCESS', changes='3,1'),
            dict(name='project-test1', result='ABORTED', changes='2,1'),
            dict(name='project-test2', result='ABORTED', changes='2,1'),
            dict(name='project1-project2-integration',
                 result='ABORTED', changes='2,1'),
            dict(name='project-test1', result='ABORTED', changes='3,1'),
            dict(name='project-test2', result='ABORTED', changes='3,1'),
            dict(name='project1-project2-integration',
                 result='ABORTED', changes='3,1'),
            dict(name='project-test1', result='ABORTED', changes='1,1'),
            dict(name='project-test2', result='ABORTED', changes='1,1'),
        ], ordered=False)

        tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
        self.assertIsNone(tenant)

    @simple_layout("layouts/reconfigure-failed-head.yaml")
    def test_live_reconfiguration_failed_change_at_head(self):
        # Test that if we reconfigure with a failed change at head,
        # that the change behind it isn't reparented onto it.

        self.executor_server.hold_jobs_in_build = True
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        A.addApproval('Code-Review', 2)
        B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
        B.addApproval('Code-Review', 2)

        self.executor_server.failJob('job1', A)

        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.fake_gerrit.addEvent(B.addApproval('Approved', 1))

        self.waitUntilSettled()

        self.assertBuilds([
            dict(name='job1', changes='1,1'),
            dict(name='job2', changes='1,1'),
            dict(name='job1', changes='1,1 2,1'),
            dict(name='job2', changes='1,1 2,1'),
        ])

        self.release(self.builds[0])
        self.waitUntilSettled()

        self.assertBuilds([
            dict(name='job2', changes='1,1'),
            dict(name='job1', changes='2,1'),
            dict(name='job2', changes='2,1'),
        ])

        # Unordered history comparison because the aborts can finish
        # in any order.
        self.assertHistory([
            dict(name='job1', result='FAILURE', changes='1,1'),
            dict(name='job1', result='ABORTED', changes='1,1 2,1'),
            dict(name='job2', result='ABORTED', changes='1,1 2,1'),
        ], ordered=False)

        self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
        self.waitUntilSettled()

        self.executor_server.hold_jobs_in_build = False
        self.executor_server.release()
        self.waitUntilSettled()

        self.assertBuilds([])

        self.assertHistory([
            dict(name='job1', result='FAILURE', changes='1,1'),
            dict(name='job1', result='ABORTED', changes='1,1 2,1'),
            dict(name='job2', result='ABORTED', changes='1,1 2,1'),
            dict(name='job2', result='SUCCESS', changes='1,1'),
            dict(name='job1', result='SUCCESS', changes='2,1'),
            dict(name='job2', result='SUCCESS', changes='2,1'),
        ], ordered=False)
        self.assertEqual(A.data['status'], 'NEW')
        self.assertEqual(B.data['status'], 'MERGED')
        self.assertEqual(A.reported, 2)
        self.assertEqual(B.reported, 2)

    def test_delayed_repo_init(self):
        self.init_repo("org/new-project")
        files = {'README': ''}
        self.addCommitToRepo("org/new-project", 'Initial commit',
                             files=files, tag='init')
        self.newTenantConfig('tenants/delayed-repo-init.yaml')
        self.commitConfigUpdate(
            'common-config',
            'layouts/delayed-repo-init.yaml')
        self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
        self.waitUntilSettled()

        A = self.fake_gerrit.addFakeChange('org/new-project', 'master', 'A')

        A.addApproval('Code-Review', 2)
        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.waitUntilSettled()
        self.assertEqual(self.getJobFromHistory('project-merge').result,
                         'SUCCESS')
        self.assertEqual(self.getJobFromHistory('project-test1').result,
                         'SUCCESS')
        self.assertEqual(self.getJobFromHistory('project-test2').result,
                         'SUCCESS')
        self.assertEqual(A.data['status'], 'MERGED')
        self.assertEqual(A.reported, 2)

    @simple_layout('layouts/single-job-with-nodeset.yaml')
    def test_live_reconfiguration_queued_node_requests(self):
        # Test that a job with a queued node request still has the
        # correct state after reconfiguration.
        self.fake_nodepool.pause()
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        def get_job():
            tenant = self.scheds.first.sched.abide.tenants['tenant-one']
            for manager in tenant.layout.pipeline_managers.values():
                pipeline_status = manager.formatStatusJSON(
                    self.scheds.first.sched.globals.websocket_url)
                for queue in pipeline_status['change_queues']:
                    for head in queue['heads']:
                        for item in head:
                            for job in item['jobs']:
                                if job['name'] == 'check-job':
                                    return job

        job = get_job()
        self.assertTrue(job['queued'])

        self.scheds.execute(lambda app: app.sched.reconfigure(self.config))
        self.waitUntilSettled()

        job = get_job()
        self.assertTrue(job['queued'])

        self.fake_nodepool.unpause()
        self.waitUntilSettled()
        self.assertHistory([
            dict(name='check-job', result='SUCCESS', changes='1,1'),
        ])

    @simple_layout('layouts/trigger-sequence.yaml')
    def test_live_reconfiguration_trigger_sequence(self):
        # Test that events arriving after an event that triggers a
        # reconfiguration are handled after the reconfiguration
        # completes.

        in_repo_conf = "[{project: {tag: {jobs: [post-job]}}}]"
        file_dict = {'zuul.yaml': in_repo_conf}
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
                                           files=file_dict)
        sched = self.scheds.first.sched
        # Hold the management queue so that we don't process any
        # reconfiguration events yet.
        with management_queue_lock(self.zk_client, 'tenant-one'):
            with sched.run_handler_lock:
                A.setMerged()
                # Submit two events while no processing is happening:
                # A change merged event that will trigger a reconfiguration
                self.fake_gerrit.addEvent(A.getChangeMergedEvent())

                # And a tag event which should only run a job after
                # the config change above is in effect.
                event = self.fake_gerrit.addFakeTag(
                    'org/project', 'master', 'foo')
                self.fake_gerrit.addEvent(event)

            # Wait for the tenant trigger queue to empty out, and for
            # us to have a tenant management as well as a pipeline
            # trigger event.  At this point, we should be deferring
            # the trigger event until the management event is handled.
            for _ in iterate_timeout(60, 'queues'):
                with sched.run_handler_lock:
                    if sched.trigger_events['tenant-one'].hasEvents():
                        continue
                    if not sched.pipeline_trigger_events[
                            'tenant-one']['tag'].hasEvents():
                        continue
                    if not sched.management_events['tenant-one'].hasEvents():
                        continue
                    break

        # Now we can resume and process the reconfiguration event
        sched.wake_event.set()
        self.waitUntilSettled()
        self.assertHistory([
            dict(name='post-job', result='SUCCESS'),
        ])

    @simple_layout('layouts/repo-deleted.yaml')
    def test_repo_deleted(self):
        self.init_repo("org/delete-project")
        A = self.fake_gerrit.addFakeChange('org/delete-project', 'master', 'A')

        A.addApproval('Code-Review', 2)
        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.waitUntilSettled()
        self.assertEqual(self.getJobFromHistory('project-merge').result,
                         'SUCCESS')
        self.assertEqual(self.getJobFromHistory('project-test1').result,
                         'SUCCESS')
        self.assertEqual(self.getJobFromHistory('project-test2').result,
                         'SUCCESS')
        self.assertEqual(A.data['status'], 'MERGED')
        self.assertEqual(A.reported, 2)

        # Delete org/new-project zuul repo. Should be recloned.
        p = 'org/delete-project'
        if os.path.exists(os.path.join(self.merger_src_root, p)):
            shutil.rmtree(os.path.join(self.merger_src_root, p))
        if os.path.exists(os.path.join(self.executor_src_root, p)):
            shutil.rmtree(os.path.join(self.executor_src_root, p))

        B = self.fake_gerrit.addFakeChange('org/delete-project', 'master', 'B')

        B.addApproval('Code-Review', 2)
        self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
        self.waitUntilSettled()
        self.assertEqual(self.getJobFromHistory('project-merge').result,
                         'SUCCESS')
        self.assertEqual(self.getJobFromHistory('project-test1').result,
                         'SUCCESS')
        self.assertEqual(self.getJobFromHistory('project-test2').result,
                         'SUCCESS')
        self.assertEqual(B.data['status'], 'MERGED')
        self.assertEqual(B.reported, 2)

    @simple_layout('layouts/untrusted-secrets.yaml')
    def test_untrusted_secrets(self):
        "Test untrusted secrets"
        A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        self.assertHistory([])
        self.assertEqual(A.patchsets[0]['approvals'][0]['value'], "-1")
        self.assertIn('does not allow post-review job',
                      A.messages[0])

    @simple_layout('layouts/tags.yaml')
    def test_tags(self):
        "Test job tags"
        A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
        B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        self.assertEqual(len(self.history), 2)

        results = {self.getJobFromHistory('merge',
                   project='org/project1').uuid: ['extratag', 'merge'],
                   self.getJobFromHistory('merge',
                   project='org/project2').uuid: ['merge']}

        for build in self.history:
            self.assertEqual(results.get(build.uuid, ''),
                             build.parameters['zuul'].get('jobtags'))

    def test_timer_template(self):
        "Test that a periodic job is triggered"
        # This test can not use simple_layout because it must start
        # with a configuration which does not include a
        # timer-triggered job so that we have an opportunity to set
        # the hold flag before the first job.
        self.create_branch('org/project', 'stable')
        self.fake_gerrit.addEvent(
            self.fake_gerrit.getFakeBranchCreatedEvent(
                'org/project', 'stable'))
        self.executor_server.hold_jobs_in_build = True
        self.commitConfigUpdate('common-config', 'layouts/timer-template.yaml')
        self.scheds.execute(lambda app: app.sched.reconfigure(app.config))

        # The pipeline triggers every second, so we should have seen
        # several by now.
        time.sleep(5)
        self.waitUntilSettled()

        self.assertEqual(len(self.builds), 2)

        merge_count_project1 = 0
        for job in self.merge_job_history.get(
            zuul.model.MergeRequest.REF_STATE
        ):
            if job.payload["items"][0]["project"] == "org/project1":
                merge_count_project1 += 1
        self.assertEqual(merge_count_project1, 0,
                         "project1 shouldn't have any refstate call")

        self.executor_server.hold_jobs_in_build = False
        # Stop queuing timer triggered jobs so that the assertions
        # below don't race against more jobs being queued.
        self.commitConfigUpdate('common-config', 'layouts/no-timer.yaml')
        self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
        self.waitUntilSettled()
        # If APScheduler is in mid-event when we remove the job, we
        # can end up with one more event firing, so give it an extra
        # second to settle.
        time.sleep(1)
        self.waitUntilSettled()
        self.executor_server.release()
        self.waitUntilSettled()

        self.assertHistory([
            dict(name='project-bitrot', result='SUCCESS',
                 ref='refs/heads/master'),
            dict(name='project-bitrot', result='SUCCESS',
                 ref='refs/heads/stable'),
        ], ordered=False)

    def _test_timer(self, config_file):
        # This test can not use simple_layout because it must start
        # with a configuration which does not include a
        # timer-triggered job so that we have an opportunity to set
        # the hold flag before the first job.
        timer = self.scheds.first.sched.connections.drivers['timer']
        start_jobs = timer.apsched.get_jobs()

        self.create_branch('org/project', 'stable')
        self.fake_gerrit.addEvent(
            self.fake_gerrit.getFakeBranchCreatedEvent(
                'org/project', 'stable'))
        self.executor_server.hold_jobs_in_build = True
        self.commitConfigUpdate('common-config', config_file)
        self.scheds.execute(lambda app: app.sched.reconfigure(app.config))

        first_jobs = timer.apsched.get_jobs()
        # Collect the currently cached branches in order to later check,
        # that the timer driver refreshes the cache.
        cached_versions = {}
        tenant = self.scheds.first.sched.abide.tenants['tenant-one']
        for project_name in tenant.layout.project_configs:
            _, project = tenant.getProject('org/project')
            for branch in project.source.getProjectBranches(project, tenant):
                event = self._create_dummy_event(project, branch)
                change_key = project.source.getChangeKey(event)
                change = project.source.getChange(change_key, event=event)
                cached_versions[branch] = change.cache_version

        # The pipeline triggers every second, so we should have seen
        # several by now.
        for _ in iterate_timeout(60, 'jobs started'):
            if len(self.builds) > 1:
                break

        # Ensure that the status json has the ref so we can render it in the
        # web ui.
        tenant = self.scheds.first.sched.abide.tenants['tenant-one']
        manager = tenant.layout.pipeline_managers['periodic']
        pipeline_status = manager.formatStatusJSON(
            self.scheds.first.sched.globals.websocket_url)

        first = pipeline_status['change_queues'][0]['heads'][0][0]
        second = pipeline_status['change_queues'][1]['heads'][0][0]
        self.assertIn(first['refs'][0]['ref'],
                      ['refs/heads/master', 'refs/heads/stable'])
        self.assertIn(second['refs'][0]['ref'],
                      ['refs/heads/master', 'refs/heads/stable'])

        self.executor_server.hold_jobs_in_build = False

        # Stop queuing timer triggered jobs so that the assertions
        # below don't race against more jobs being queued.
        self.commitConfigUpdate('common-config', 'layouts/no-timer.yaml')
        self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
        second_jobs = timer.apsched.get_jobs()
        self.waitUntilSettled()
        # If APScheduler is in mid-event when we remove the job, we
        # can end up with one more event firing, so give it an extra
        # second to settle.
        time.sleep(3)
        self.waitUntilSettled()

        self.executor_server.release()
        self.waitUntilSettled()

        self.assertTrue(len(self.history) > 1)
        for job in self.history[:1]:
            self.assertEqual(job.result, 'SUCCESS')
            self.assertEqual(job.name, 'project-bitrot')
            self.assertIn(job.ref, ('refs/heads/stable', 'refs/heads/master'))

        for project_name in tenant.layout.project_configs:
            _, project = tenant.getProject('org/project')
            for branch in project.source.getProjectBranches(project, tenant):
                event = self._create_dummy_event(project, branch)
                change_key = project.source.getChangeKey(event)
                change = project.source.getChange(change_key, event=event)
                # Make sure the timer driver refreshed the cache
                self.assertGreater(change.cache_version,
                                   cached_versions[branch])

        # We start with no jobs, and our first reconfigure should add jobs
        self.assertTrue(len(first_jobs) > len(start_jobs))
        # Our second reconfigure should return us to no jobs
        self.assertEqual(start_jobs, second_jobs)

    def _create_dummy_event(self, project, branch):
        event = zuul.model.TriggerEvent()
        event.type = 'test'
        event.project_hostname = project.canonical_hostname
        event.project_name = project.name
        event.ref = f'refs/heads/{branch}'
        event.branch = branch
        event.zuul_event_id = str(uuid4().hex)
        event.timestamp = time.time()
        return event

    def test_timer(self):
        "Test that a periodic job is triggered"
        self._test_timer('layouts/timer.yaml')

    def test_timer_with_jitter(self):
        "Test that a periodic job with a jitter is triggered"
        self._test_timer('layouts/timer-jitter.yaml')

    def test_timer_preserve_jobs(self):
        # This tests that we keep the same apsched jobs if possible
        # when reconfiguring.  If a reconfiguration happens during the
        # "jitter" period, we might end up not running jobs unless we
        # preserve the exact job object across reconfiguration.
        self.commitConfigUpdate('common-config', 'layouts/timer-jitter.yaml')
        self.scheds.execute(lambda app: app.sched.reconfigure(app.config))

        timer = self.scheds.first.sched.connections.drivers['timer']
        old_jobs = timer.apsched.get_jobs()

        self.scheds.execute(lambda app: app.sched.reconfigure(app.config))

        new_jobs = timer.apsched.get_jobs()

        self.assertEqual(old_jobs, new_jobs)

        # Stop queuing timer triggered jobs so that the final
        # assertions don't race against more jobs being queued.
        self.commitConfigUpdate('common-config', 'layouts/no-timer.yaml')
        self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
        self.waitUntilSettled()
        # If APScheduler is in mid-event when we remove the job, we
        # can end up with one more event firing, so give it an extra
        # second to settle.
        time.sleep(3)
        self.waitUntilSettled()

    def test_idle(self):
        "Test that frequent periodic jobs work"
        # This test can not use simple_layout because it must start
        # with a configuration which does not include a
        # timer-triggered job so that we have an opportunity to set
        # the hold flag before the first job.
        self.executor_server.hold_jobs_in_build = True

        for x in range(1, 3):
            # Test that timer triggers periodic jobs even across
            # layout config reloads.
            # Start timer trigger
            self.commitConfigUpdate('common-config',
                                    'layouts/idle.yaml')
            self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
            self.waitUntilSettled()

            # The pipeline triggers every second, so we should have seen
            # several by now.
            time.sleep(5)

            # Stop queuing timer triggered jobs so that the assertions
            # below don't race against more jobs being queued.
            self.commitConfigUpdate('common-config',
                                    'layouts/no-timer.yaml')
            self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
            self.waitUntilSettled()
            # If APScheduler is in mid-event when we remove the job,
            # we can end up with one more event firing, so give it an
            # extra second to settle.
            time.sleep(1)
            self.waitUntilSettled()
            self.assertEqual(len(self.builds), 1,
                             'Timer builds iteration #%d' % x)
            self.executor_server.release('.*')
            self.waitUntilSettled()
            self.assertEqual(len(self.builds), 0)
            self.assertEqual(len(self.history), x)

    @simple_layout('layouts/smtp.yaml')
    def test_check_smtp_pool(self):
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        self.waitUntilSettled()

        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        self.assertEqual(len(self.smtp_messages), 2)

        # A.messages only holds what FakeGerrit places in it. Thus we
        # work on the knowledge of what the first message should be as
        # it is only configured to go to SMTP.

        self.assertEqual('zuul@example.com',
                         self.smtp_messages[0]['from_email'])
        self.assertEqual(['you@example.com'],
                         self.smtp_messages[0]['to_email'])
        self.assertEqual('Starting check jobs.',
                         self.smtp_messages[0]['body'])

        self.assertEqual('zuul_from@example.com',
                         self.smtp_messages[1]['from_email'])
        self.assertEqual(['alternative_me@example.com'],
                         self.smtp_messages[1]['to_email'])
        self.assertEqual(A.messages[0],
                         self.smtp_messages[1]['body'])

    @simple_layout('layouts/smtp.yaml')
    @mock.patch('zuul.driver.gerrit.gerritreporter.GerritReporter.report')
    @okay_tracebacks('Gerrit failed to report')
    def test_failed_reporter(self, report_mock):
        '''Test that one failed reporter doesn't break other reporters'''
        # Warning hacks. We sort the reports here so that the test is
        # deterministic. Gerrit reporting will fail, but smtp reporting
        # should succeed.
        report_mock.side_effect = Exception('Gerrit failed to report')

        tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
        check = tenant.layout.pipeline_managers['check'].pipeline

        check.success_actions = sorted(check.success_actions,
                                       key=lambda x: x.name)
        self.assertEqual(check.success_actions[0].name, 'gerrit')
        self.assertEqual(check.success_actions[1].name, 'smtp')

        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        self.waitUntilSettled()
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        # We know that if gerrit ran first and failed and smtp ran second
        # and sends mail then we handle failures in reporters gracefully.
        self.assertEqual(len(self.smtp_messages), 2)

        # A.messages only holds what FakeGerrit places in it. Thus we
        # work on the knowledge of what the first message should be as
        # it is only configured to go to SMTP.

        self.assertEqual('zuul@example.com',
                         self.smtp_messages[0]['from_email'])
        self.assertEqual(['you@example.com'],
                         self.smtp_messages[0]['to_email'])
        self.assertEqual('Starting check jobs.',
                         self.smtp_messages[0]['body'])

        self.assertEqual('zuul_from@example.com',
                         self.smtp_messages[1]['from_email'])
        self.assertEqual(['alternative_me@example.com'],
                         self.smtp_messages[1]['to_email'])
        # This double checks that Gerrit side failed
        self.assertEqual(A.messages, [])

    def test_timer_smtp(self):
        "Test that a periodic job is triggered"
        # This test can not use simple_layout because it must start
        # with a configuration which does not include a
        # timer-triggered job so that we have an opportunity to set
        # the hold flag before the first job.
        self.executor_server.hold_jobs_in_build = True
        self.commitConfigUpdate('common-config', 'layouts/timer-smtp.yaml')
        self.scheds.execute(lambda app: app.sched.reconfigure(app.config))

        # The pipeline triggers every second, so we should have seen
        # several by now.
        time.sleep(5)
        self.waitUntilSettled()

        self.assertEqual(len(self.builds), 2)
        self.executor_server.release('.*')
        self.waitUntilSettled()
        self.assertEqual(len(self.history), 2)

        self.assertEqual(self.getJobFromHistory(
            'project-bitrot-stable-old').result, 'SUCCESS')
        self.assertEqual(self.getJobFromHistory(
            'project-bitrot-stable-older').result, 'SUCCESS')

        self.assertEqual(len(self.smtp_messages), 1)

        # A.messages only holds what FakeGerrit places in it. Thus we
        # work on the knowledge of what the first message should be as
        # it is only configured to go to SMTP.

        self.assertEqual('zuul_from@example.com',
                         self.smtp_messages[0]['from_email'])
        self.assertEqual(['alternative_me@example.com'],
                         self.smtp_messages[0]['to_email'])
        self.assertIn('Subject: Periodic check for org/project succeeded',
                      self.smtp_messages[0]['headers'])

        # Stop queuing timer triggered jobs and let any that may have
        # queued through so that end of test assertions pass.
        self.commitConfigUpdate('common-config', 'layouts/no-timer.yaml')
        self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
        self.waitUntilSettled()
        # If APScheduler is in mid-event when we remove the job, we
        # can end up with one more event firing, so give it an extra
        # second to settle.
        time.sleep(1)
        self.waitUntilSettled()
        self.executor_server.release('.*')
        self.waitUntilSettled()

    @skip("Disabled for early v3 development")
    def test_timer_sshkey(self):
        "Test that a periodic job can setup SSH key authentication"
        self.worker.hold_jobs_in_build = True
        self.config.set('zuul', 'layout_config',
                        'tests/fixtures/layout-timer.yaml')
        self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
        self.registerJobs()

        # The pipeline triggers every second, so we should have seen
        # several by now.
        time.sleep(5)
        self.waitUntilSettled()

        self.assertEqual(len(self.builds), 2)

        ssh_wrapper = os.path.join(self.git_root, ".ssh_wrapper_gerrit")
        self.assertTrue(os.path.isfile(ssh_wrapper))
        with open(ssh_wrapper) as f:
            ssh_wrapper_content = f.read()
        self.assertIn("fake_id_rsa", ssh_wrapper_content)
        # In the unit tests Merger runs in the same process,
        # so we see its' environment variables
        self.assertEqual(os.environ['GIT_SSH'], ssh_wrapper)

        self.worker.release('.*')
        self.waitUntilSettled()
        self.assertEqual(len(self.history), 2)

        self.assertEqual(self.getJobFromHistory(
            'project-bitrot-stable-old').result, 'SUCCESS')
        self.assertEqual(self.getJobFromHistory(
            'project-bitrot-stable-older').result, 'SUCCESS')

        # Stop queuing timer triggered jobs and let any that may have
        # queued through so that end of test assertions pass.
        self.config.set('zuul', 'layout_config',
                        'tests/fixtures/layout-no-timer.yaml')
        self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
        self.registerJobs()
        self.waitUntilSettled()
        # If APScheduler is in mid-event when we remove the job, we
        # can end up with one more event firing, so give it an extra
        # second to settle.
        time.sleep(1)
        self.waitUntilSettled()
        self.worker.release('.*')
        self.waitUntilSettled()

    @simple_layout('layouts/rate-limit.yaml')
    def test_queue_rate_limiting(self):
        "Test that DependentPipelines are rate limited with dep across window"
        self.executor_server.hold_jobs_in_build = True
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
        C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')

        C.setDependsOn(B, 1)
        self.executor_server.failJob('project-test1', A)

        A.addApproval('Code-Review', 2)
        B.addApproval('Code-Review', 2)
        C.addApproval('Code-Review', 2)

        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
        self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
        self.waitUntilSettled()

        # Only A and B will have their merge jobs queued because
        # window is 2.
        self.assertEqual(len(self.builds), 2)
        self.assertEqual(self.builds[0].name, 'project-merge')
        self.assertEqual(self.builds[1].name, 'project-merge')

        # Release the merge jobs one at a time.
        self.builds[0].release()
        self.waitUntilSettled()
        self.builds[0].release()
        self.waitUntilSettled()

        # Only A and B will have their test jobs queued because
        # window is 2.
        self.assertEqual(len(self.builds), 4)
        self.assertEqual(self.builds[0].name, 'project-test1')
        self.assertEqual(self.builds[1].name, 'project-test2')
        self.assertEqual(self.builds[2].name, 'project-test1')
        self.assertEqual(self.builds[3].name, 'project-test2')

        self.executor_server.release('project-.*')
        self.waitUntilSettled()

        tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
        queue = tenant.layout.pipeline_managers['gate'].state.queues[0]
        # A failed so window is reduced by 1 to 1.
        self.assertEqual(queue.window, 1)
        self.assertEqual(queue.window_floor, 1)
        self.assertEqual(queue.window_ceiling, 2)
        self.assertEqual(A.data['status'], 'NEW')

        # Gate is reset and only B's merge job is queued because
        # window shrunk to 1.
        self.assertEqual(len(self.builds), 1)
        self.assertEqual(self.builds[0].name, 'project-merge')

        self.executor_server.release('.*-merge')
        self.waitUntilSettled()

        # Only B's test jobs are queued because window is still 1.
        self.assertEqual(len(self.builds), 2)
        self.assertEqual(self.builds[0].name, 'project-test1')
        self.assertEqual(self.builds[1].name, 'project-test2')

        self.executor_server.release('project-.*')
        self.waitUntilSettled()

        # B was successfully merged so window is increased to 2.
        self.assertEqual(queue.window, 2)
        self.assertEqual(queue.window_floor, 1)
        self.assertEqual(queue.window_ceiling, 2)
        self.assertEqual(B.data['status'], 'MERGED')

        # Only C is left and its merge job is queued.
        self.assertEqual(len(self.builds), 1)
        self.assertEqual(self.builds[0].name, 'project-merge')

        self.executor_server.release('.*-merge')
        self.waitUntilSettled()

        # After successful merge job the test jobs for C are queued.
        self.assertEqual(len(self.builds), 2)
        self.assertEqual(self.builds[0].name, 'project-test1')
        self.assertEqual(self.builds[1].name, 'project-test2')

        self.executor_server.release('project-.*')
        self.waitUntilSettled()

        # C successfully merged but hit the ceiling of 2
        self.assertEqual(queue.window, 2)
        self.assertEqual(queue.window_floor, 1)
        self.assertEqual(queue.window_ceiling, 2)
        self.assertEqual(C.data['status'], 'MERGED')

    @simple_layout('layouts/rate-limit.yaml')
    def test_queue_rate_limiting_dependent(self):
        "Test that DependentPipelines are rate limited with dep in window"
        self.executor_server.hold_jobs_in_build = True
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
        C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')

        B.setDependsOn(A, 1)

        self.executor_server.failJob('project-test1', A)

        A.addApproval('Code-Review', 2)
        B.addApproval('Code-Review', 2)
        C.addApproval('Code-Review', 2)

        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
        self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
        self.waitUntilSettled()

        # Only A and B will have their merge jobs queued because
        # window is 2.
        self.assertEqual(len(self.builds), 2)
        self.assertEqual(self.builds[0].name, 'project-merge')
        self.assertEqual(self.builds[1].name, 'project-merge')

        self.orderedRelease(2)

        # Only A and B will have their test jobs queued because
        # window is 2.
        self.assertEqual(len(self.builds), 4)
        self.assertEqual(self.builds[0].name, 'project-test1')
        self.assertEqual(self.builds[1].name, 'project-test2')
        self.assertEqual(self.builds[2].name, 'project-test1')
        self.assertEqual(self.builds[3].name, 'project-test2')

        self.executor_server.release('project-.*')
        self.waitUntilSettled()

        tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
        queue = tenant.layout.pipeline_managers['gate'].state.queues[0]
        # A failed so window is reduced by 1 to 1.
        self.assertEqual(queue.window, 1)
        self.assertEqual(queue.window_floor, 1)
        self.assertEqual(A.data['status'], 'NEW')
        self.assertEqual(B.data['status'], 'NEW')

        # Gate is reset and only C's merge job is queued because
        # window shrunk to 1 and A and B were dequeued.
        self.assertEqual(len(self.builds), 1)
        self.assertEqual(self.builds[0].name, 'project-merge')

        self.orderedRelease(1)

        # Only C's test jobs are queued because window is still 1.
        self.assertEqual(len(self.builds), 2)
        builds = self.getSortedBuilds()
        self.assertEqual(builds[0].name, 'project-test1')
        self.assertEqual(builds[1].name, 'project-test2')

        self.executor_server.release('project-.*')
        self.waitUntilSettled()

        # C was successfully merged so window is increased to 2.
        self.assertEqual(queue.window, 2)
        self.assertEqual(queue.window_floor, 1)
        self.assertEqual(C.data['status'], 'MERGED')

    @simple_layout('layouts/rate-limit-reconfigure.yaml')
    def test_queue_rate_limiting_reconfigure(self):
        """Test that changes survive a reconfigure when no longer in window.

        This is a regression tests for a case that lead to an exception during
        re-enqueue. The exception happened when former active items had already
        build results but then dropped out of the active window. During
        re-enqueue the job graph was not re-initialized because the items were
        no longer active.
        """
        self.executor_server.hold_jobs_in_build = True
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
        C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
        D = self.fake_gerrit.addFakeChange('org/project', 'master', 'D')

        A.addApproval('Code-Review', 2)
        B.addApproval('Code-Review', 2)
        C.addApproval('Code-Review', 2)
        D.addApproval('Code-Review', 2)

        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
        self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
        self.fake_gerrit.addEvent(D.addApproval('Approved', 1))
        self.waitUntilSettled()

        self.assertEqual(len(self.builds), 4)
        self.assertEqual(self.builds[0].name, 'project-merge')
        self.assertEqual(self.builds[1].name, 'project-merge')
        self.assertEqual(self.builds[2].name, 'project-merge')
        self.assertEqual(self.builds[3].name, 'project-merge')

        self.orderedRelease(4)

        self.assertEqual(len(self.builds), 8)
        self.assertEqual(self.builds[0].name, 'project-test1')
        self.assertEqual(self.builds[1].name, 'project-test2')
        self.assertEqual(self.builds[2].name, 'project-test1')
        self.assertEqual(self.builds[3].name, 'project-test2')
        self.assertEqual(self.builds[4].name, 'project-test1')
        self.assertEqual(self.builds[5].name, 'project-test2')
        self.assertEqual(self.builds[6].name, 'project-test1')
        self.assertEqual(self.builds[7].name, 'project-test2')

        self.executor_server.failJob('project-test1', B)
        self.builds[2].release()
        self.builds[3].release()
        self.waitUntilSettled()

        self.assertEqual(len(self.builds), 4)
        # A's jobs
        self.assertEqual(self.builds[0].name, 'project-test1')
        self.assertEqual(self.builds[1].name, 'project-test2')
        # C's and D's merge jobs
        self.assertEqual(self.builds[2].name, 'project-merge')
        self.assertEqual(self.builds[3].name, 'project-merge')

        # Release merge jobs of C, D after speculative gate reset
        self.executor_server.release('project-merge')
        self.waitUntilSettled()

        self.assertEqual(len(self.builds), 6)

        # A's jobs
        self.assertEqual(self.builds[0].name, 'project-test1')
        self.assertEqual(self.builds[1].name, 'project-test2')
        # C's + D's jobs
        self.assertEqual(self.builds[2].name, 'project-test1')
        self.assertEqual(self.builds[3].name, 'project-test2')
        self.assertEqual(self.builds[4].name, 'project-test1')
        self.assertEqual(self.builds[5].name, 'project-test2')

        # Fail D's job so we have a build results for an item that
        # is not in the active window after B is reported
        # (condition that previously lead to an exception)
        self.executor_server.failJob('project-test1', D)
        self.builds[4].release()
        self.waitUntilSettled()

        # Release A's jobs
        self.builds[0].release()
        self.builds[1].release()
        self.waitUntilSettled()

        self.assertEqual(len(self.builds), 3)

        # C's jobs
        self.assertEqual(self.builds[0].name, 'project-test1')
        self.assertEqual(self.builds[1].name, 'project-test2')
        # D's remaining job
        self.assertEqual(self.builds[2].name, 'project-test2')

        tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
        queue = tenant.layout.pipeline_managers['gate'].state.queues[0]
        self.assertEqual(queue.window, 1)

        # D dropped out of the window
        self.assertFalse(queue.queue[-1].active)

        self.commitConfigUpdate('org/common-config',
                                'layouts/rate-limit-reconfigure2.yaml')
        self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
        self.waitUntilSettled()

        # D's remaining job should still be queued
        self.assertEqual(len(self.builds), 3)
        self.executor_server.release('project-.*')
        self.waitUntilSettled()

    @simple_layout('layouts/reconfigure-window.yaml')
    def test_reconfigure_window_shrink(self):
        # Test the active window shrinking during reconfiguration
        self.executor_server.hold_jobs_in_build = True

        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        A.addApproval('Code-Review', 2)
        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.waitUntilSettled()
        B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
        B.addApproval('Code-Review', 2)
        self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
        self.waitUntilSettled()

        tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
        queue = tenant.layout.pipeline_managers['gate'].state.queues[0]
        self.assertEqual(queue.window, 20)
        self.assertTrue(len(self.builds), 4)

        self.executor_server.release('job1')
        self.waitUntilSettled()
        self.commitConfigUpdate('org/common-config',
                                'layouts/reconfigure-window2.yaml')
        self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
        self.waitUntilSettled()
        tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
        queue = tenant.layout.pipeline_managers['gate'].state.queues[0]
        # Even though we have configured a smaller window, the value
        # on the existing shared queue should be used.
        self.assertEqual(queue.window, 20)
        self.assertTrue(len(self.builds), 4)

        self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
        self.waitUntilSettled()
        tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
        queue = tenant.layout.pipeline_managers['gate'].state.queues[0]
        self.assertEqual(queue.window, 20)
        self.assertTrue(len(self.builds), 4)

        self.executor_server.hold_jobs_in_build = False
        self.executor_server.release()

        self.waitUntilSettled()
        self.assertHistory([
            dict(name='job1', result='SUCCESS', changes='1,1'),
            dict(name='job1', result='SUCCESS', changes='1,1 2,1'),
            dict(name='job2', result='SUCCESS', changes='1,1'),
            dict(name='job2', result='SUCCESS', changes='1,1 2,1'),
        ], ordered=False)

    @simple_layout('layouts/reconfigure-window-fixed.yaml')
    def test_reconfigure_window_fixed(self):
        # Test the active window shrinking during reconfiguration
        self.executor_server.hold_jobs_in_build = True

        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        A.addApproval('Code-Review', 2)
        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.waitUntilSettled()
        B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
        B.addApproval('Code-Review', 2)
        self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
        self.waitUntilSettled()

        tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
        queue = tenant.layout.pipeline_managers['gate'].state.queues[0]
        self.assertEqual(queue.window, 2)
        self.assertEqual(len(self.builds), 4)

        self.waitUntilSettled()
        self.commitConfigUpdate('org/common-config',
                                'layouts/reconfigure-window-fixed2.yaml')
        self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
        self.waitUntilSettled()
        tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
        queue = tenant.layout.pipeline_managers['gate'].state.queues[0]
        # Because we have configured a static window, it should
        # be allowed to shrink on reconfiguration.
        self.assertEqual(queue.window, 1)
        # B is outside the window, but still marked active until the
        # next pass through the queue processor.
        self.assertEqual(len(self.builds), 4)

        self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
        self.waitUntilSettled()
        tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
        queue = tenant.layout.pipeline_managers['gate'].state.queues[0]
        self.assertEqual(queue.window, 1)
        self.waitUntilSettled()
        # B's builds should not be canceled
        self.assertEqual(len(self.builds), 4)

        self.executor_server.hold_jobs_in_build = False
        self.executor_server.release()

        self.waitUntilSettled()
        self.assertHistory([
            dict(name='job1', result='SUCCESS', changes='1,1'),
            dict(name='job2', result='SUCCESS', changes='1,1'),
            dict(name='job1', result='SUCCESS', changes='1,1 2,1'),
            dict(name='job2', result='SUCCESS', changes='1,1 2,1'),
        ], ordered=False)

    @simple_layout('layouts/reconfigure-window-fixed.yaml')
    def test_reconfigure_window_fixed_requests(self):
        # Test the active window shrinking during reconfiguration with
        # outstanding node requests
        self.executor_server.hold_jobs_in_build = True

        # Start the jobs for A
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        A.addApproval('Code-Review', 2)
        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.waitUntilSettled()
        self.log.debug("A complete")

        # Hold node requests for B
        self.fake_nodepool.pause()
        B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
        B.addApproval('Code-Review', 2)
        self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
        self.waitUntilSettled()
        self.log.debug("B complete")

        tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
        queue = tenant.layout.pipeline_managers['gate'].state.queues[0]
        self.assertEqual(queue.window, 2)
        self.assertEqual(len(self.builds), 2)

        self.waitUntilSettled()
        self.commitConfigUpdate('org/common-config',
                                'layouts/reconfigure-window-fixed2.yaml')
        self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
        self.waitUntilSettled()
        self.log.debug("Reconfiguration complete")

        tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
        queue = tenant.layout.pipeline_managers['gate'].state.queues[0]
        # Because we have configured a static window, it should
        # be allowed to shrink on reconfiguration.
        self.assertEqual(queue.window, 1)
        self.assertEqual(len(self.builds), 2)

        # After the previous reconfig, the queue processor will have
        # run and marked B inactive; run another reconfiguration so
        # that we're testing what happens when we reconfigure after
        # the active window having shrunk.
        self.scheds.execute(lambda app: app.sched.reconfigure(app.config))

        # Unpause the node requests now
        self.fake_nodepool.unpause()
        self.waitUntilSettled()
        self.log.debug("Nodepool unpause complete")

        # Allow A to merge and B to enter the active window and complete
        self.executor_server.hold_jobs_in_build = False
        self.executor_server.release()
        self.waitUntilSettled()
        self.log.debug("Executor unpause complete")

        tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
        queue = tenant.layout.pipeline_managers['gate'].state.queues[0]
        self.assertEqual(queue.window, 1)

        self.waitUntilSettled()
        self.assertHistory([
            dict(name='job1', result='SUCCESS', changes='1,1'),
            dict(name='job2', result='SUCCESS', changes='1,1'),
            dict(name='job1', result='SUCCESS', changes='1,1 2,1'),
            dict(name='job2', result='SUCCESS', changes='1,1 2,1'),
        ], ordered=False)

    @simple_layout('layouts/footer-message.yaml')
    def test_footer_message(self):
        "Test a pipeline's footer message is correctly added to the report."
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        A.addApproval('Code-Review', 2)
        self.executor_server.failJob('project-test1', A)
        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.waitUntilSettled()

        B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
        B.addApproval('Code-Review', 2)
        self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
        self.waitUntilSettled()

        self.assertEqual(2, len(self.smtp_messages))

        failure_msg = """\
Build failed.  For information on how to proceed, see \
http://wiki.example.org/Test_Failures"""

        footer_msg = """\
For CI problems and help debugging, contact ci@example.org"""

        self.assertTrue(self.smtp_messages[0]['body'].startswith(failure_msg))
        self.assertTrue(self.smtp_messages[0]['body'].endswith(footer_msg))
        self.assertFalse(self.smtp_messages[1]['body'].startswith(failure_msg))
        self.assertTrue(self.smtp_messages[1]['body'].endswith(footer_msg))

    @simple_layout('layouts/start-message.yaml')
    def test_start_message(self):
        "Test a pipeline's start message is correctly added to the report."
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        A.addApproval('Code-Review', 2)
        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.waitUntilSettled()

        self.assertEqual(1, len(self.smtp_messages))
        start_msg = (
            "Jobs started in gate: "
            "https://zuul.example.com/t/tenant-one/status/change/1,1."
        )
        self.assertTrue(self.smtp_messages[0]['body'].startswith(start_msg))

    @simple_layout('layouts/unmanaged-project.yaml')
    def test_unmanaged_project_start_message(self):
        "Test start reporting is not done for unmanaged projects."
        self.init_repo("org/project", tag='init')
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        self.assertEqual(0, len(A.messages))

    @simple_layout('layouts/merge-conflict.yaml')
    def test_merge_conflict_reporters(self):
        """Check that the config is set up correctly"""

        tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
        self.assertEqual(
            "Merge Failed.\n\nThis change or one of its cross-repo "
            "dependencies was unable to be automatically merged with the "
            "current state of its repository. Please rebase the change and "
            "upload a new patchset.",
            tenant.layout.pipeline_managers['check'].
            pipeline.merge_conflict_message)
        self.assertEqual(
            "The merge failed! For more information...",
            tenant.layout.pipeline_managers['gate'].
            pipeline.merge_conflict_message)

        self.assertEqual(
            len(tenant.layout.pipeline_managers['check'].
                pipeline.merge_conflict_actions), 1)
        self.assertEqual(
            len(tenant.layout.pipeline_managers['gate'].
                pipeline.merge_conflict_actions), 2)

        self.assertTrue(isinstance(
            tenant.layout.pipeline_managers['check'].
            pipeline.merge_conflict_actions[0],
            gerritreporter.GerritReporter))

        self.assertTrue(
            (
                isinstance(tenant.layout.pipeline_managers['gate'].
                           pipeline.merge_conflict_actions[0],
                           zuul.driver.smtp.smtpreporter.SMTPReporter) and
                isinstance(tenant.layout.pipeline_managers['gate'].
                           pipeline.merge_conflict_actions[1],
                           gerritreporter.GerritReporter)
            ) or (
                isinstance(tenant.layout.pipeline_managers['gate'].
                           pipeline.merge_conflict_actions[0],
                           gerritreporter.GerritReporter) and
                isinstance(tenant.layout.pipeline_managers['gate'].
                           pipeline.merge_conflict_actions[1],
                           zuul.driver.smtp.smtpreporter.SMTPReporter)
            )
        )

    @simple_layout('layouts/merge-failure.yaml')
    def test_merge_failure_reporters(self):
        """Check that the config is set up correctly"""
        # TODO: Remove this backwards compat test in v6.0

        tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
        self.assertEqual(
            "Merge Failed.\n\nThis change or one of its cross-repo "
            "dependencies was unable to be automatically merged with the "
            "current state of its repository. Please rebase the change and "
            "upload a new patchset.",
            tenant.layout.pipeline_managers['check'].
            pipeline.merge_conflict_message)
        self.assertEqual(
            "The merge failed! For more information...",
            tenant.layout.pipeline_managers['gate'].
            pipeline.merge_conflict_message)

        self.assertEqual(
            len(tenant.layout.pipeline_managers['check'].
                pipeline.merge_conflict_actions), 1)
        self.assertEqual(
            len(tenant.layout.pipeline_managers['gate'].
                pipeline.merge_conflict_actions), 2)

        self.assertTrue(isinstance(
            tenant.layout.pipeline_managers['check'].
            pipeline.merge_conflict_actions[0],
            gerritreporter.GerritReporter))

        self.assertTrue(
            (
                isinstance(tenant.layout.pipeline_managers['gate'].
                           pipeline.merge_conflict_actions[0],
                           zuul.driver.smtp.smtpreporter.SMTPReporter) and
                isinstance(tenant.layout.pipeline_managers['gate'].
                           pipeline.merge_conflict_actions[1],
                           gerritreporter.GerritReporter)
            ) or (
                isinstance(tenant.layout.pipeline_managers['gate'].
                           pipeline.merge_conflict_actions[0],
                           gerritreporter.GerritReporter) and
                isinstance(tenant.layout.pipeline_managers['gate'].
                           pipeline.merge_conflict_actions[1],
                           zuul.driver.smtp.smtpreporter.SMTPReporter)
            )
        )

    def test_merge_failure_reports(self):
        """Check that when a change fails to merge the correct message is sent
        to the correct reporter"""
        self.commitConfigUpdate('common-config',
                                'layouts/merge-conflict.yaml')
        self.scheds.execute(lambda app: app.sched.reconfigure(app.config))

        # Check a test failure isn't reported to SMTP
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        A.addApproval('Code-Review', 2)
        self.executor_server.failJob('project-test1', A)
        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.waitUntilSettled()

        self.assertEqual(3, len(self.history))  # 3 jobs
        self.assertEqual(0, len(self.smtp_messages))

        # Check a merge failure is reported to SMTP
        # B should be merged, but C will conflict with B
        B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
        B.addPatchset({'conflict': 'foo'})
        C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
        C.addPatchset({'conflict': 'bar'})
        B.addApproval('Code-Review', 2)
        C.addApproval('Code-Review', 2)
        self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
        self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
        self.waitUntilSettled()

        self.assertEqual(6, len(self.history))  # A and B jobs
        self.assertEqual(1, len(self.smtp_messages))
        self.assertIn('The merge failed! For more information...',
                      self.smtp_messages[0]['body'])
        self.assertIn('Error merging gerrit/org/project',
                      self.smtp_messages[0]['body'])

    def test_default_merge_failure_reports(self):
        """Check that the default merge failure reports are correct."""

        # A should report success, B should report merge failure.
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        A.addPatchset({'conflict': 'foo'})
        B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
        B.addPatchset({'conflict': 'bar'})
        A.addApproval('Code-Review', 2)
        B.addApproval('Code-Review', 2)
        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
        self.waitUntilSettled()

        self.assertEqual(3, len(self.history))  # A jobs
        self.assertEqual(A.reported, 2)
        self.assertEqual(B.reported, 1)
        self.assertEqual(A.data['status'], 'MERGED')
        self.assertEqual(B.data['status'], 'NEW')
        self.assertIn('Build succeeded', A.messages[1])
        self.assertIn('Merge Failed', B.messages[0])
        self.assertIn('automatically merged', B.messages[0])
        self.assertIn('Error merging gerrit/org/project', B.messages[0])
        self.assertNotIn('logs.example.com', B.messages[0])
        self.assertNotIn('SKIPPED', B.messages[0])
        buildsets = list(
            self.scheds.first.connections.connections[
                'database'].getBuildsets())
        self.assertEqual(buildsets[0].result, 'MERGE_CONFLICT')
        self.assertIn('This change or one of', buildsets[0].message)

    def test_submit_failure(self):
        A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
        A.fail_merge = True

        A.addApproval('Code-Review', 2)
        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.waitUntilSettled()

        buildsets = list(
            self.scheds.first.connections.connections[
                'database'].getBuildsets())
        self.assertEqual(buildsets[0].result, 'MERGE_FAILURE')

    @simple_layout('layouts/timer-freeze-job-failure.yaml')
    def test_periodic_freeze_job_failure(self):
        self.waitUntilSettled()

        for x in iterate_timeout(30, 'buildset complete'):
            buildsets = list(
                self.scheds.first.connections.connections[
                    'database'].getBuildsets())
            if buildsets:
                break
        # Stop queuing timer triggered jobs so that the assertions
        # below don't race against more jobs being queued.
        self.commitConfigUpdate('org/common-config', 'layouts/no-timer.yaml')
        self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
        self.waitUntilSettled()
        # If APScheduler is in mid-event when we remove the job, we
        # can end up with one more event firing, so give it an extra
        # second to settle.
        time.sleep(3)
        self.waitUntilSettled()

        self.assertEqual(buildsets[0].result, 'CONFIG_ERROR')
        self.assertIn('Job project-test2 depends on project-test1 '
                      'which was not run', buildsets[0].message)

    @simple_layout('layouts/freeze-job-failure.yaml')
    def test_freeze_job_failure(self):
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        buildsets = list(
            self.scheds.first.connections.connections[
                'database'].getBuildsets())
        self.assertEqual(buildsets[0].result, 'CONFIG_ERROR')
        self.assertIn('Job project-test2 depends on project-test1 '
                      'which was not run', buildsets[0].message)

    @simple_layout('layouts/nonvoting-pipeline.yaml')
    def test_nonvoting_pipeline(self):
        "Test that a nonvoting pipeline (experimental) can still report"

        A = self.fake_gerrit.addFakeChange('org/experimental-project',
                                           'master', 'A')
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        self.assertEqual(self.getJobFromHistory('project-merge').result,
                         'SUCCESS')
        self.assertEqual(
            self.getJobFromHistory('experimental-project-test').result,
            'SUCCESS')
        self.assertEqual(A.reported, 1)

    @simple_layout('layouts/disable_at.yaml')
    def test_disable_at(self):
        "Test a pipeline will only report to the disabled trigger when failing"

        tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
        self.assertEqual(
            3, tenant.layout.pipeline_managers['check'].pipeline.disable_at)
        self.assertEqual(
            0,
            tenant.layout.pipeline_managers[
                'check'].state.consecutive_failures)
        self.assertFalse(
            tenant.layout.pipeline_managers['check'].state.disabled)

        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
        C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
        D = self.fake_gerrit.addFakeChange('org/project', 'master', 'D')
        E = self.fake_gerrit.addFakeChange('org/project', 'master', 'E')
        F = self.fake_gerrit.addFakeChange('org/project', 'master', 'F')
        G = self.fake_gerrit.addFakeChange('org/project', 'master', 'G')
        H = self.fake_gerrit.addFakeChange('org/project', 'master', 'H')
        I = self.fake_gerrit.addFakeChange('org/project', 'master', 'I')
        J = self.fake_gerrit.addFakeChange('org/project', 'master', 'J')
        K = self.fake_gerrit.addFakeChange('org/project', 'master', 'K')

        self.executor_server.failJob('project-test1', A)
        self.executor_server.failJob('project-test1', B)
        # Let C pass, resetting the counter
        self.executor_server.failJob('project-test1', D)
        self.executor_server.failJob('project-test1', E)
        self.executor_server.failJob('project-test1', F)
        self.executor_server.failJob('project-test1', G)
        self.executor_server.failJob('project-test1', H)
        # I also passes but should only report to the disabled reporters
        self.executor_server.failJob('project-test1', J)
        self.executor_server.failJob('project-test1', K)

        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        self.assertEqual(
            2,
            tenant.layout.pipeline_managers[
                'check'].state.consecutive_failures)
        self.assertFalse(
            tenant.layout.pipeline_managers['check'].state.disabled)

        self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        self.assertEqual(
            0,
            tenant.layout.pipeline_managers[
                'check'].state.consecutive_failures)
        self.assertFalse(
            tenant.layout.pipeline_managers['check'].state.disabled)

        self.fake_gerrit.addEvent(D.getPatchsetCreatedEvent(1))
        self.fake_gerrit.addEvent(E.getPatchsetCreatedEvent(1))
        self.fake_gerrit.addEvent(F.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        # We should be disabled now
        self.assertEqual(
            3,
            tenant.layout.pipeline_managers[
                'check'].state.consecutive_failures)
        self.assertTrue(
            tenant.layout.pipeline_managers['check'].state.disabled)

        # We need to wait between each of these patches to make sure the
        # smtp messages come back in an expected order
        self.fake_gerrit.addEvent(G.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()
        self.fake_gerrit.addEvent(H.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()
        self.fake_gerrit.addEvent(I.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        # The first 6 (ABCDEF) jobs should have reported back to gerrt thus
        # leaving a message on each change
        self.assertEqual(1, len(A.messages))
        self.assertIn('Build failed.', A.messages[0])
        self.assertEqual(1, len(B.messages))
        self.assertIn('Build failed.', B.messages[0])
        self.assertEqual(1, len(C.messages))
        self.assertIn('Build succeeded.', C.messages[0])
        self.assertEqual(1, len(D.messages))
        self.assertIn('Build failed.', D.messages[0])
        self.assertEqual(1, len(E.messages))
        self.assertIn('Build failed.', E.messages[0])
        self.assertEqual(1, len(F.messages))
        self.assertIn('Build failed.', F.messages[0])

        # The last 3 (GHI) would have only reported via smtp.
        self.assertEqual(3, len(self.smtp_messages))
        self.assertEqual(0, len(G.messages))
        self.assertIn('Build failed.', self.smtp_messages[0]['body'])
        self.assertIn(
            'project-test1 https://', self.smtp_messages[0]['body'])
        self.assertEqual(0, len(H.messages))
        self.assertIn('Build failed.', self.smtp_messages[1]['body'])
        self.assertIn(
            'project-test1 https://', self.smtp_messages[1]['body'])
        self.assertEqual(0, len(I.messages))
        self.assertIn('Build succeeded.', self.smtp_messages[2]['body'])
        self.assertIn(
            'project-test1 https://', self.smtp_messages[2]['body'])

        # Now reload the configuration (simulate a HUP) to check the pipeline
        # comes out of disabled
        self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
        self.waitUntilSettled()

        tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')

        self.assertEqual(
            3, tenant.layout.pipeline_managers['check'].pipeline.disable_at)
        self.assertEqual(
            0,
            tenant.layout.pipeline_managers[
                'check'].state.consecutive_failures)
        self.assertFalse(
            tenant.layout.pipeline_managers['check'].state.disabled)

        self.fake_gerrit.addEvent(J.getPatchsetCreatedEvent(1))
        self.fake_gerrit.addEvent(K.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        self.assertEqual(
            2,
            tenant.layout.pipeline_managers[
                'check'].state.consecutive_failures)
        self.assertFalse(
            tenant.layout.pipeline_managers['check'].state.disabled)

        # J and K went back to gerrit
        self.assertEqual(1, len(J.messages))
        self.assertIn('Build failed.', J.messages[0])
        self.assertEqual(1, len(K.messages))
        self.assertIn('Build failed.', K.messages[0])
        # No more messages reported via smtp
        self.assertEqual(3, len(self.smtp_messages))

    @simple_layout('layouts/one-job-project.yaml')
    def test_one_job_project(self):
        "Test that queueing works with one job"
        A = self.fake_gerrit.addFakeChange('org/one-job-project',
                                           'master', 'A')
        B = self.fake_gerrit.addFakeChange('org/one-job-project',
                                           'master', 'B')
        A.addApproval('Code-Review', 2)
        B.addApproval('Code-Review', 2)
        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
        self.waitUntilSettled()

        self.assertEqual(A.data['status'], 'MERGED')
        self.assertEqual(A.reported, 2)
        self.assertEqual(B.data['status'], 'MERGED')
        self.assertEqual(B.reported, 2)

    def test_job_aborted(self):
        "Test that if a execute server aborts a job, it is run again"
        self.executor_server.hold_jobs_in_build = True
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        self.executor_server.release('.*-merge')
        self.waitUntilSettled()

        self.assertEqual(len(self.builds), 2)

        # first abort
        self.builds[0].aborted = True
        self.executor_server.release('.*-test*')
        self.waitUntilSettled()
        self.assertEqual(len(self.builds), 1)

        # second abort
        self.builds[0].aborted = True
        self.executor_server.release('.*-test*')
        self.waitUntilSettled()
        self.assertEqual(len(self.builds), 1)

        # third abort
        self.builds[0].aborted = True
        self.executor_server.release('.*-test*')
        self.waitUntilSettled()
        self.assertEqual(len(self.builds), 1)

        # fourth abort
        self.builds[0].aborted = True
        self.executor_server.release('.*-test*')
        self.waitUntilSettled()
        self.assertEqual(len(self.builds), 1)

        self.executor_server.hold_jobs_in_build = False
        self.executor_server.release()
        self.waitUntilSettled()

        self.assertEqual(len(self.history), 7)
        self.assertEqual(self.countJobResults(self.history, 'ABORTED'), 4)
        self.assertEqual(self.countJobResults(self.history, 'SUCCESS'), 3)

    def test_rerun_on_abort(self):
        "Test that if a execute server fails to run a job, it is run again"

        self.executor_server.hold_jobs_in_build = True
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        self.executor_server.release('.*-merge')
        self.waitUntilSettled()

        self.assertEqual(len(self.builds), 2)
        self.builds[0].requeue = True
        self.executor_server.release('.*-test*')
        self.waitUntilSettled()

        items = self.getAllItems('tenant-one', 'check')
        build_set = items[0].current_build_set
        job = list(filter(lambda j: j.name == 'project-test1',
                          items[0].getJobs()))[0]

        for x in range(3):
            # We should have x+1 retried builds for project-test1
            retry_builds = build_set.getRetryBuildsForJob(job)
            self.assertEqual(len(retry_builds), x + 1)
            for build in retry_builds:
                self.assertEqual(build.retry, True)
                self.assertEqual(build.result, 'RETRY')

            self.assertEqual(len(self.builds), 1,
                             'len of builds at x=%d is wrong' % x)
            self.builds[0].requeue = True
            self.executor_server.release('.*-test1')
            self.waitUntilSettled()

        self.executor_server.hold_jobs_in_build = False
        self.executor_server.release()
        self.waitUntilSettled()
        self.assertEqual(len(self.history), 6)
        self.assertEqual(self.countJobResults(self.history, 'SUCCESS'), 2)
        self.assertEqual(A.reported, 1)
        self.assertIn('RETRY_LIMIT', A.messages[0])

    def test_executor_disconnect(self):
        "Test that jobs are completed after an executor disconnect"

        self.executor_server.hold_jobs_in_build = True
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        A.addApproval('Code-Review', 2)
        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.waitUntilSettled()

        # Forcibly disconnect the executor from ZK
        self.executor_server.zk_client.client.stop()
        self.executor_server.zk_client.client.start()

        # Find the build in the scheduler so we can check its status
        items = self.getAllItems('tenant-one', 'gate')
        builds = items[0].current_build_set.getBuilds()
        build = builds[0]

        # Clean up the build
        self.scheds.first.sched.executor.cleanupLostBuildRequests()
        # Wait for the build to be reported as lost
        for x in iterate_timeout(30, 'retry build'):
            if build.result == 'RETRY':
                break

        # If we didn't timeout, then it worked; we're done
        self.executor_server.hold_jobs_in_build = False
        self.executor_server.release()
        self.waitUntilSettled()

        # There is a test-only race in the recording executor class
        # where we may record a successful first build, even though
        # the executor didn't actually send a build complete event.
        # This could probabyl be improved, but for now, it's
        # sufficient to verify that the job was retried.  So we omit a
        # result classifier on the first build.
        self.assertHistory([
            dict(name='project-merge', changes='1,1'),
            dict(name='project-merge', result='SUCCESS', changes='1,1'),
            dict(name='project-test1', result='SUCCESS', changes='1,1'),
            dict(name='project-test2', result='SUCCESS', changes='1,1'),
        ], ordered=False)

    # TODO: There seems to be a race condition in the kazoo election
    # recipe that can cause the stats election thread to hang after
    # reconnecting.
    @skip("This is unstable in the gate")
    def test_scheduler_disconnect(self):
        "Test that jobs are completed after a scheduler disconnect"

        self.executor_server.hold_jobs_in_build = True
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        A.addApproval('Code-Review', 2)
        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.waitUntilSettled()

        # Forcibly disconnect the scheduler from ZK
        self.scheds.execute(lambda app: app.sched.zk_client.client.stop())
        self.scheds.execute(lambda app: app.sched.zk_client.client.start())

        # Clean up lost builds
        self.scheds.first.sched.executor.cleanupLostBuildRequests()

        self.executor_server.hold_jobs_in_build = False
        self.executor_server.release()
        self.waitUntilSettled()

        self.assertHistory([
            dict(name='project-merge', result='SUCCESS', changes='1,1'),
            dict(name='project-test1', result='SUCCESS', changes='1,1'),
            dict(name='project-test2', result='SUCCESS', changes='1,1'),
        ], ordered=False)

    # TODO: See comment for test_scheduler_disconnect.
    @skip("This is unstable in the gate")
    def test_zookeeper_disconnect(self):
        "Test that jobs are executed after a zookeeper disconnect"

        self.fake_nodepool.pause()
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        A.addApproval('Code-Review', 2)
        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.waitUntilSettled()

        self.scheds.execute(lambda app: app.sched.zk_client.client.stop())
        self.scheds.execute(lambda app: app.sched.zk_client.client.start())
        self.fake_nodepool.unpause()
        # Wait until we win the nodepool election in order to avoid a
        # race in waitUntilSettled with the request being fulfilled
        # without submitting an event.
        for x in iterate_timeout(60, 'nodepool election won'):
            found = [app for app in self.scheds
                     if (app.sched.nodepool.election_won and
                         app.sched.nodepool.election.is_still_valid())]
            if found:
                break
        self.waitUntilSettled()

        self.assertEqual(A.data['status'], 'MERGED')
        self.assertEqual(A.reported, 2)

    def test_nodepool_cleanup(self):
        "Test that we cleanup leaked node requests"
        self.fake_nodepool.pause()
        system_id = self.scheds.first.sched.system.system_id
        zk_nodepool = self.scheds.first.sched.nodepool.zk_nodepool
        req1 = zuul.model.NodeRequest(system_id, "uuid1", "tenant",
                                      "pipeline", "job_uuid", "job",
                                      ['label'], None, 0, None)
        zk_nodepool.submitNodeRequest(req1, 100)
        req2 = zuul.model.NodeRequest("someone else", "uuid1", "tenant",
                                      "pipeline", "job_uuid", "job",
                                      ['label'], None, 0, None)
        zk_nodepool.submitNodeRequest(req2, 100)
        self.assertEqual(zk_nodepool.getNodeRequests(),
                         ['100-0000000000', '100-0000000001'])
        self.scheds.first.sched._runNodeRequestCleanup()
        self.assertEqual(zk_nodepool.getNodeRequests(),
                         ['100-0000000001'])
        zk_nodepool.deleteNodeRequest(req2.id)

    @simple_layout('layouts/simple.yaml')
    def test_nodeset_request_cleanup(self):
        "Test that we cleanup leaked nodeset requests"

        with self.launcher._run_lock:
            with self.scheds.first.sched.node_request_cleanup_lock:
                ctx = self.createZKContext(None)
                zuul.model.NodesetRequest.new(
                    ctx,
                    tenant_name="tenant-one",
                    pipeline_name="test",
                    buildset_uuid=uuid4().hex,
                    job_uuid=uuid4().hex,
                    job_name="foobar",
                    labels=["debian-normal"],
                    priority=100,
                    request_time=time.time(),
                    zuul_event_id=uuid4().hex,
                    span_info=None,
                )
                ids = self.scheds.first.sched.launcher.getRequestIds()
                self.assertEqual(1, len(ids))
            self.scheds.first.sched._runNodeRequestCleanup()
            ids = self.scheds.first.sched.launcher.getRequestIds()
            self.assertEqual(0, len(ids))

    def test_nodepool_failure(self):
        "Test that jobs are reported after a nodepool failure"

        self.fake_nodepool.pause()
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        A.addApproval('Code-Review', 2)
        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.waitUntilSettled()

        req = self.fake_nodepool.getNodeRequests()[0]
        self.fake_nodepool.addFailRequest(req)

        self.fake_nodepool.unpause()
        self.waitUntilSettled()

        self.assertEqual(A.data['status'], 'NEW')
        self.assertEqual(A.reported, 2)
        self.assertTrue(re.search('project-merge .* NODE_FAILURE',
                                  A.messages[1]))
        self.assertTrue(
            'Skipped due to failed job project-merge' in A.messages[1])
        self.assertReportedStat(
            'zuul.nodepool.requests.requested.total', value='1', kind='c')
        self.assertReportedStat(
            'zuul.nodepool.requests.requested.label.label1',
            value='1', kind='c')
        self.assertReportedStat(
            'zuul.nodepool.requests.failed.label.label1',
            value='1', kind='c')
        self.assertReportedStat(
            'zuul.tenant.tenant-one.pipeline.gate.project.review_example_com.'
            'org_project.master.job.project-merge.NODE_FAILURE', value='1',
            kind='c')

    def test_nodepool_resources(self):
        "Test that resources are reported"

        self.executor_server.hold_jobs_in_build = True
        self.fake_nodepool.resources = {
            'cores': 2,
            'ram': 1024,
            'instances': 1,
        }
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        A.addApproval('Code-Review', 2)
        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.waitUntilSettled()

        self.executor_server.release('project-merge')
        self.waitUntilSettled()
        self.waitUntilNodeCacheSync(
            self.scheds.first.sched.nodepool.zk_nodepool)
        self.scheds.first.sched._runStats()

        # Check that resource usage gauges are reported
        self.assertHistory([
            dict(name='project-merge', result='SUCCESS', changes='1,1'),
        ])
        self.assertReportedStat(
            'zuul.nodepool.resources.total.tenant.tenant-one.cores',
            value='6', kind='g')
        self.assertReportedStat(
            'zuul.nodepool.resources.total.tenant.tenant-one.ram',
            value='3072', kind='g')
        self.assertReportedStat(
            'zuul.nodepool.resources.total.tenant.tenant-one.instances',
            value='3', kind='g')
        # All 3 nodes are in use
        self.assertReportedStat(
            'zuul.nodepool.resources.in_use.tenant.tenant-one.cores',
            value='6', kind='g')
        self.assertReportedStat(
            'zuul.nodepool.resources.in_use.tenant.tenant-one.ram',
            value='3072', kind='g')
        self.assertReportedStat(
            'zuul.nodepool.resources.in_use.tenant.tenant-one.instances',
            value='3', kind='g')
        self.assertReportedStat(
            'zuul.nodepool.resources.in_use.project.review_example_com/org/'
            'project.cores', value='6', kind='g')
        self.assertReportedStat(
            'zuul.nodepool.resources.in_use.project.review_example_com/org/'
            'project.ram', value='3072', kind='g')
        self.assertReportedStat(
            'zuul.nodepool.resources.in_use.project.review_example_com/org/'
            'project.instances', value='3', kind='g')

        # Check that resource usage counters are reported
        self.assertReportedStat(
            'zuul.nodepool.resources.in_use.tenant.tenant-one.cores',
            kind='c')
        self.assertReportedStat(
            'zuul.nodepool.resources.in_use.tenant.tenant-one.ram',
            kind='c')
        self.assertReportedStat(
            'zuul.nodepool.resources.in_use.tenant.tenant-one.instances',
            kind='c')
        self.assertReportedStat(
            'zuul.nodepool.resources.in_use.project.review_example_com/org/'
            'project.cores', kind='c')
        self.assertReportedStat(
            'zuul.nodepool.resources.in_use.project.review_example_com/org/'
            'project.ram', kind='c')
        self.assertReportedStat(
            'zuul.nodepool.resources.in_use.project.review_example_com/org/'
            'project.instances', kind='c')

        self.executor_server.hold_jobs_in_build = False
        self.executor_server.release()
        self.waitUntilSettled()
        self.assertEqual(A.data['status'], 'MERGED')
        self.assertEqual(A.reported, 2)

    def test_nodepool_pipeline_priority(self):
        "Test that nodes are requested at the correct pipeline priority"

        self.fake_nodepool.pause()

        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        A.setMerged()
        self.fake_gerrit.addEvent(A.getRefUpdatedEvent())
        self.waitUntilSettled()

        B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
        C.addApproval('Code-Review', 2)
        self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
        self.waitUntilSettled()

        reqs = self.fake_nodepool.getNodeRequests()

        # The requests come back sorted by priority. Since we have
        # three requests for the three changes each with a different
        # priority.  Also they get a serial number based on order they
        # were received so the number on the endof the oid should map
        # to order submitted.

        # * gate first - high priority - change C
        self.assertEqual(reqs[0]['_oid'], '100-0000000002')
        self.assertEqual(reqs[0]['node_types'], ['label1'])
        # * check second - normal priority - change B
        self.assertEqual(reqs[1]['_oid'], '200-0000000001')
        self.assertEqual(reqs[1]['node_types'], ['label1'])
        # * post third - low priority - change A
        # additionally, the post job defined uses an ubuntu-xenial node,
        # so we include that check just as an extra verification
        self.assertEqual(reqs[2]['_oid'], '300-0000000000')
        self.assertEqual(reqs[2]['node_types'], ['ubuntu-xenial'])

        self.fake_nodepool.unpause()
        self.waitUntilSettled()
        self.assertHistory([
            dict(name='project-merge', result='SUCCESS', changes='2,1'),
            dict(name='project-test1', result='SUCCESS', changes='2,1'),
            dict(name='project-test2', result='SUCCESS', changes='2,1'),
            dict(name='project-merge', result='SUCCESS', changes='3,1'),
            dict(name='project-test1', result='SUCCESS', changes='3,1'),
            dict(name='project-test2', result='SUCCESS', changes='3,1'),
            dict(name='project1-project2-integration',
                 result='SUCCESS', changes='2,1'),
            dict(name='project-post', result='SUCCESS'),
        ], ordered=False)

    @simple_layout('layouts/two-projects-integrated.yaml')
    def test_nodepool_relative_priority_check(self):
        "Test that nodes are requested at the relative priority"

        self.fake_nodepool.pause()

        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        C = self.fake_gerrit.addFakeChange('org/project1', 'master', 'C')
        self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        D = self.fake_gerrit.addFakeChange('org/project2', 'master', 'D')
        self.fake_gerrit.addEvent(D.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        reqs = self.fake_nodepool.getNodeRequests()

        # The requests come back sorted by priority.

        # Change A, first change for project, high relative priority.
        self.assertEqual(reqs[0]['_oid'], '200-0000000000')
        self.assertEqual(reqs[0]['relative_priority'], 0)

        # Change C, first change for project1, high relative priority.
        self.assertEqual(reqs[1]['_oid'], '200-0000000002')
        self.assertEqual(reqs[1]['relative_priority'], 0)

        # Change B, second change for project, lower relative priority.
        self.assertEqual(reqs[2]['_oid'], '200-0000000001')
        self.assertEqual(reqs[2]['relative_priority'], 1)

        # Change D, first change for project2 shared with project1,
        # lower relative priority than project1.
        self.assertEqual(reqs[3]['_oid'], '200-0000000003')
        self.assertEqual(reqs[3]['relative_priority'], 1)

        # Fulfill only the first request
        self.fake_nodepool.fulfillRequest(reqs[0])
        for x in iterate_timeout(30, 'fulfill request'):
            reqs = list(self.scheds.first.sched.nodepool.getNodeRequests())
            if len(reqs) < 4:
                break
        self.waitUntilSettled()

        reqs = self.fake_nodepool.getNodeRequests()

        # Change B, now first change for project, equal priority.
        self.assertEqual(reqs[0]['_oid'], '200-0000000001')
        self.assertEqual(reqs[0]['relative_priority'], 0)

        # Change C, now first change for project1, equal priority.
        self.assertEqual(reqs[1]['_oid'], '200-0000000002')
        self.assertEqual(reqs[1]['relative_priority'], 0)

        # Change D, first change for project2 shared with project1,
        # still lower relative priority than project1.
        self.assertEqual(reqs[2]['_oid'], '200-0000000003')
        self.assertEqual(reqs[2]['relative_priority'], 1)

        self.fake_nodepool.unpause()
        self.waitUntilSettled()

    @simple_layout('layouts/two-projects-integrated.yaml')
    def test_nodepool_relative_priority_long(self):
        "Test that nodes are requested at the relative priority"

        self.fake_nodepool.pause()

        count = 13
        changes = []
        for x in range(count):
            change = self.fake_gerrit.addFakeChange(
                'org/project', 'master', 'A')
            self.fake_gerrit.addEvent(change.getPatchsetCreatedEvent(1))
            self.waitUntilSettled()
            changes.append(change)

        reqs = self.fake_nodepool.getNodeRequests()
        self.assertEqual(len(reqs), 13)
        # The requests come back sorted by priority.
        for x in range(10):
            self.assertEqual(reqs[x]['relative_priority'], x)
        self.assertEqual(reqs[10]['relative_priority'], 10)
        self.assertEqual(reqs[11]['relative_priority'], 10)
        self.assertEqual(reqs[12]['relative_priority'], 10)

        # Fulfill only the first request
        self.fake_nodepool.fulfillRequest(reqs[0])
        for x in iterate_timeout(30, 'fulfill request'):
            reqs = list(self.scheds.first.sched.nodepool.getNodeRequests())
            if len(reqs) < count:
                break
        self.waitUntilSettled()

        reqs = self.fake_nodepool.getNodeRequests()
        self.assertEqual(len(reqs), 12)
        for x in range(10):
            self.assertEqual(reqs[x]['relative_priority'], x)
        self.assertEqual(reqs[10]['relative_priority'], 10)
        self.assertEqual(reqs[11]['relative_priority'], 10)

        self.fake_nodepool.unpause()
        self.waitUntilSettled()

    @simple_layout('layouts/two-projects-integrated.yaml')
    def test_nodepool_relative_priority_gate(self):
        "Test that nodes are requested at the relative priority"

        self.fake_nodepool.pause()

        A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
        A.addApproval('Code-Review', 2)
        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.waitUntilSettled()

        B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
        B.addApproval('Code-Review', 2)
        self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
        self.waitUntilSettled()

        # project does not share a queue with project1 and project2.
        C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
        C.addApproval('Code-Review', 2)
        self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
        self.waitUntilSettled()

        reqs = self.fake_nodepool.getNodeRequests()

        # The requests come back sorted by priority.

        # Change A, first change for shared queue, high relative
        # priority.
        self.assertEqual(reqs[0]['_oid'], '100-0000000000')
        self.assertEqual(reqs[0]['relative_priority'], 0)

        # Change C, first change for independent project, high
        # relative priority.
        self.assertEqual(reqs[1]['_oid'], '100-0000000002')
        self.assertEqual(reqs[1]['relative_priority'], 0)

        # Change B, second change for shared queue, lower relative
        # priority.
        self.assertEqual(reqs[2]['_oid'], '100-0000000001')
        self.assertEqual(reqs[2]['relative_priority'], 1)

        self.fake_nodepool.unpause()
        self.waitUntilSettled()

    def test_nodepool_project_removal(self):
        "Test that nodes are returned unused after project removal"

        self.fake_nodepool.pause()
        A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        self.newTenantConfig('config/single-tenant/main-one-project.yaml')
        # This layout defines only org/project, not org/project1
        self.commitConfigUpdate(
            'common-config',
            'layouts/live-reconfiguration-del-project.yaml')
        self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
        self.waitUntilSettled()

        self.fake_nodepool.unpause()
        self.waitUntilSettled()

        self.assertEqual(A.data['status'], 'NEW')
        self.assertEqual(A.reported, 0)
        for node in self.fake_nodepool.getNodes():
            self.assertFalse(node['_lock'])
            self.assertEqual(node['state'], 'ready')

    @simple_layout('layouts/nodeset-fallback.yaml')
    def test_nodeset_fallback(self):
        # Test that nodeset fallback works
        self.executor_server.hold_jobs_in_build = True

        # Verify that we get the correct number and order of
        # alternates from our nested config.
        tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
        job = tenant.layout.getJob('check-job')
        alts = job.flattenNodesetAlternatives(tenant.layout)
        self.assertEqual(4, len(alts))
        self.assertEqual('fast-nodeset', alts[0].name)
        self.assertEqual('', alts[1].name)
        self.assertEqual('red-nodeset', alts[2].name)
        self.assertEqual('blue-nodeset', alts[3].name)

        self.fake_nodepool.pause()
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        req = self.fake_nodepool.getNodeRequests()[0]
        self.fake_nodepool.addFailRequest(req)

        self.fake_nodepool.unpause()
        self.waitUntilSettled()

        build = self.getBuildByName('check-job')
        inv_path = os.path.join(build.jobdir.root, 'ansible', 'inventory.yaml')
        with open(inv_path, 'r') as f:
            inventory = yaml.safe_load(f)
        label = inventory['all']['hosts']['controller']['nodepool']['label']
        self.assertEqual('slow-label', label)

        self.executor_server.hold_jobs_in_build = False
        self.executor_server.release()
        self.waitUntilSettled()

        self.assertEqual(A.data['status'], 'NEW')
        self.assertEqual(A.reported, 1)
        self.assertNotIn('NODE_FAILURE', A.messages[0])
        self.assertHistory([
            dict(name='check-job', result='SUCCESS', changes='1,1'),
        ], ordered=False)

    @simple_layout('layouts/multiple-templates.yaml')
    def test_multiple_project_templates(self):
        # Test that applying multiple project templates to a project
        # doesn't alter them when used for a second project.
        A = self.fake_gerrit.addFakeChange('org/project2', 'master', 'A')
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        build = self.getJobFromHistory('py27')
        self.assertEqual(build.parameters['zuul']['jobtags'], [])

    def test_pending_merge_in_reconfig(self):
        # Test that if we are waiting for an outstanding merge on
        # reconfiguration that we continue to do so.
        self.hold_merge_jobs_in_queue = True
        A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
        A.setMerged()
        self.fake_gerrit.addEvent(A.getRefUpdatedEvent())
        self.waitUntilSettled()

        jobs = list(self.merger_api.all())
        self.assertEqual(len(jobs), 1)
        self.assertEqual(jobs[0].state, zuul.model.MergeRequest.HOLD)

        # Reconfigure while we still have an outstanding merge job
        self.hold_merge_jobs_in_queue = False
        tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
        (trusted, project1) = tenant.getProject('org/project1')
        event = zuul.model.TriggerEvent()
        event.zuul_event_ltime = self.zk_client.getCurrentLtime()
        self.scheds.first.sched.reconfigureTenant(
            self.scheds.first.sched.abide.tenants['tenant-one'],
            project1, event)
        self.waitUntilSettled()

        # Verify the merge job is still running and that the item is
        # in the pipeline
        jobs = list(self.merger_api.all())
        self.assertEqual(jobs[0].state, zuul.model.MergeRequest.HOLD)
        self.assertEqual(len(jobs), 1)

        self.assertEqual(len(self.getAllItems('tenant-one', 'post')), 1)
        self.merger_api.release()
        self.waitUntilSettled()

        jobs = list(self.merger_api.all())
        self.assertEqual(len(jobs), 0)

    @simple_layout('layouts/parent-matchers.yaml')
    def test_parent_matchers(self):
        "Test that if a job's parent does not match, the job does not run"
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        self.assertHistory([])

        files = {'foo.txt': ''}
        B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B',
                                           files=files)
        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        files = {'bar.txt': ''}
        C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C',
                                           files=files)
        self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        files = {'foo.txt': '', 'bar.txt': ''}
        D = self.fake_gerrit.addFakeChange('org/project', 'master', 'D',
                                           files=files)
        self.fake_gerrit.addEvent(D.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        self.assertHistory([
            dict(name='child-job', result='SUCCESS', changes='3,1'),
            dict(name='child-job', result='SUCCESS', changes='4,1'),
        ], ordered=False)

    @simple_layout('layouts/file-matchers.yaml')
    def test_file_matchers(self):
        "Test several file matchers"
        files = {'parent1.txt': ''}
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
                                           files=files)
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        files = {'parent2.txt': ''}
        B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B',
                                           files=files)
        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        files = {'child.txt': ''}
        C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C',
                                           files=files)
        self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        files = {'project.txt': ''}
        D = self.fake_gerrit.addFakeChange('org/project', 'master', 'D',
                                           files=files)
        self.fake_gerrit.addEvent(D.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        files = {'tests/foo': ''}
        E = self.fake_gerrit.addFakeChange('org/project', 'master', 'E',
                                           files=files)
        self.fake_gerrit.addEvent(E.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        files = {'tests/docs/foo': ''}
        F = self.fake_gerrit.addFakeChange('org/project', 'master', 'F',
                                           files=files)
        self.fake_gerrit.addEvent(F.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        self.assertHistory([
            dict(name='child-job', result='SUCCESS', changes='2,1'),

            dict(name='child-override-job', result='SUCCESS', changes='3,1'),

            dict(name='project-override-job', result='SUCCESS', changes='4,1'),

            dict(name='irr-job', result='SUCCESS', changes='5,1'),
            dict(name='irr-override-job', result='SUCCESS', changes='5,1'),

            dict(name='irr-job', result='SUCCESS', changes='6,1'),
        ], ordered=False)

    def test_trusted_project_dep_on_non_live_untrusted_project(self):
        # Test we get a layout for trusted projects when they depend on
        # non live untrusted projects. This checks against a bug where
        # trusted project config changes can end up in a infinite loop
        # trying to find the right layout.
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        files = {'zuul.yaml': ''}
        B = self.fake_gerrit.addFakeChange('common-config', 'master', 'B',
                                           files=files)
        B.setDependsOn(A, 1)
        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        self.assertHistory([
            dict(name='project-merge', result='SUCCESS', changes='1,1 2,1'),
            dict(name='project-test1', result='SUCCESS', changes='1,1 2,1'),
            dict(name='project-test2', result='SUCCESS', changes='1,1 2,1'),
            dict(name='project1-project2-integration',
                 result='SUCCESS', changes='1,1 2,1'),
        ], ordered=False)

    @simple_layout('layouts/success-message.yaml')
    def test_success_message(self):
        # Test the success_message (and failure_message) job attrs
        A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
        self.executor_server.failJob('badjob', A)
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()
        self.assertEqual(len(A.messages), 1)
        self.assertTrue('YAY' in A.messages[0])
        self.assertTrue('BOO' in A.messages[0])

    @okay_tracebacks('git_fail.sh')
    def test_merge_error(self):
        # Test we don't get stuck on a merger error
        self.waitUntilSettled()
        self.patch(zuul.merger.merger.Repo, 'retry_attempts', 1)

        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        self.patch(git.Git, 'GIT_PYTHON_GIT_EXECUTABLE',
                   os.path.join(FIXTURE_DIR, 'git_fail.sh'))
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()
        self.assertIn('Unable to update gerrit/org/project', A.messages[0])

    @simple_layout('layouts/vars.yaml')
    def test_jobdata(self):
        # Test the use of JobData objects for job variables
        self.executor_server.hold_jobs_in_build = True

        self.useFixture(fixtures.MonkeyPatch(
            'zuul.model.FrozenJob.MAX_DATA_LEN',
            1))
        self.useFixture(fixtures.MonkeyPatch(
            'zuul.model.Build.MAX_DATA_LEN',
            1))

        # Return some data and pause the job.  We use a paused job
        # here because there are only two times we refresh JobData:
        # 1) A job which has not yet started its build
        #    because the waiting status may change, we refresh the FrozenJob
        # 2) A job which is paused
        #    because the result/data may change, we refresh the Build
        # This allows us to test that we re-use JobData instances when
        # we are able.
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        self.executor_server.returnData(
            'check-job', A,
            {'somedata': 'foobar',
             'zuul': {'pause': True}},
        )
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
        item = tenant.layout.pipeline_managers[
            'check'].state.queues[0].queue[0]
        hold_job = item.getJobs()[1]

        # Refresh the pipeline so that we can verify the JobData
        # objects are immutable.
        old_hold_job_variables = hold_job.variables
        ctx = self.refreshPipelines(self.scheds.first.sched)
        new_hold_job_variables = hold_job.variables

        self.executor_server.release('check-job')
        self.waitUntilSettled()
        # Waiting on hold-job now

        # Check the assertions on the hold job here so that the test
        # can fail normally if they fail (it times out otherwise due
        # to the held job).
        self.assertEqual('hold-job', hold_job.name)
        # Make sure we're really using JobData objects
        self.assertTrue(isinstance(hold_job._variables, zuul.model.JobData))
        # Make sure the same object instance is used
        self.assertIs(old_hold_job_variables, new_hold_job_variables)
        # Hopefully these asserts won't change much over time.  If
        # they don't they may be a good way for us to catch unintended
        # extra read operations.  If they change too much, they may
        # not be worth keeping and we can just remove them.
        self.assertEqual(5, ctx.cumulative_read_objects)
        self.assertEqual(5, ctx.cumulative_read_znodes)
        self.assertEqual(0, ctx.cumulative_write_objects)
        self.assertEqual(0, ctx.cumulative_write_znodes)

        check_job = item.getJobs()[0]
        self.assertEqual('check-job', check_job.name)
        self.assertTrue(isinstance(check_job._variables,
                                   zuul.model.JobData))
        check_build = item.current_build_set.getBuild(check_job)
        self.assertTrue(isinstance(check_build._result_data,
                                   zuul.model.JobData))

        # Refresh the pipeline so that we can verify the JobData
        # objects are immutable.
        old_check_build_results = check_build.result_data
        ctx = self.refreshPipelines(self.scheds.first.sched)
        new_check_build_results = check_build.result_data

        # Verify that we did not reload results
        self.assertIs(old_check_build_results, new_check_build_results)

        # Again check the object read counts
        self.assertEqual(4, ctx.cumulative_read_objects)
        self.assertEqual(4, ctx.cumulative_read_znodes)
        self.assertEqual(0, ctx.cumulative_write_objects)
        self.assertEqual(0, ctx.cumulative_write_znodes)

        self.executor_server.hold_jobs_in_build = False
        self.executor_server.release()
        self.waitUntilSettled()

        self.assertHistory([
            dict(name='check-job', result='SUCCESS', changes='1,1'),
            dict(name='hold-job', result='SUCCESS', changes='1,1'),
        ], ordered=False)

    def test_zkobject_parallel_refresh(self):
        # Test that we don't deadlock when refreshing objects
        zkobject.BaseZKContext._max_workers = 1

        self.executor_server.hold_jobs_in_build = True
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
        A.addApproval('Code-Review', 2)
        B.addApproval('Code-Review', 2)
        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
        self.waitUntilSettled()
        self.executor_server.hold_jobs_in_build = False
        self.executor_server.release()
        self.waitUntilSettled()

        self.assertHistory([
            dict(name='project-merge', result='SUCCESS', changes='1,1'),
            dict(name='project-test1', result='SUCCESS', changes='1,1'),
            dict(name='project-test2', result='SUCCESS', changes='1,1'),
            dict(name='project-merge', result='SUCCESS', changes='1,1 2,1'),
            dict(name='project-test1', result='SUCCESS', changes='1,1 2,1'),
            dict(name='project-test2', result='SUCCESS', changes='1,1 2,1'),
        ], ordered=False)
        self.assertEqual(A.data['status'], 'MERGED')
        self.assertEqual(B.data['status'], 'MERGED')

    def test_leaked_pipeline_cleanup(self):
        self.waitUntilSettled()
        sched = self.scheds.first.sched

        pipeline_state_path = "/zuul/tenant/tenant-one/pipeline/invalid"
        self.zk_client.client.ensure_path(pipeline_state_path)

        # Create the ZK path as a side-effect of getting the event queue.
        sched.pipeline_management_events["tenant-one"]["invalid"]
        pipeline_event_queue_path = PIPELINE_NAME_ROOT.format(
            tenant="tenant-one", pipeline="invalid")

        self.assertIsNotNone(self.zk_client.client.exists(pipeline_state_path))
        # Wait for the event watcher to create the event queues
        for _ in iterate_timeout(30, "create event queues"):
            for event_queue in ("management", "trigger", "result"):
                if self.zk_client.client.exists(
                        f"{pipeline_event_queue_path}/{event_queue}") is None:
                    break
            else:
                break

        sched._runLeakedPipelineCleanup()
        self.assertIsNone(
            self.zk_client.client.exists(pipeline_event_queue_path))
        self.assertIsNone(self.zk_client.client.exists(pipeline_state_path))


class TestChangeQueues(ZuulTestCase):
    tenant_config_file = 'config/change-queues/main.yaml'

    def _test_dependent_queues_per_branch(self, project,
                                          queue_name='integrated',
                                          queue_repo='common-config'):
        self.create_branch(project, 'stable')
        self.fake_gerrit.addEvent(
            self.fake_gerrit.getFakeBranchCreatedEvent(project, 'stable'))
        self.waitUntilSettled()

        self.executor_server.hold_jobs_in_build = True
        A = self.fake_gerrit.addFakeChange(project, 'master', 'A')
        B = self.fake_gerrit.addFakeChange(project, 'stable', 'B')
        A.addApproval('Code-Review', 2)
        B.addApproval('Code-Review', 2)

        self.executor_server.failJob('project-test', A)

        # Let first go A into gate then B
        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.waitUntilSettled()

        self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
        self.waitUntilSettled()

        # There should be one project-test job at the head of each queue
        self.assertBuilds([
            dict(name='project-test', changes='1,1'),
            dict(name='project-test', changes='2,1'),
        ])
        tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
        _, p = tenant.getProject(project)
        q1 = tenant.layout.pipeline_managers['gate'].state.getQueue(
            p.canonical_name, 'master')
        q2 = tenant.layout.pipeline_managers['gate'].state.getQueue(
            p.canonical_name, 'stable')
        self.assertEqual(q1.name, queue_name)
        self.assertEqual(q2.name, queue_name)

        # Both queues must contain one item
        self.assertEqual(len(q1.queue), 1)
        self.assertEqual(len(q2.queue), 1)

        # Fail job on the change on master
        self.executor_server.hold_jobs_in_build = False
        self.executor_server.release()
        self.waitUntilSettled()

        self.assertNotEqual(A.data['status'], 'MERGED')
        self.assertEqual(B.data['status'], 'MERGED')
        self.assertEqual(A.reported, 2)
        self.assertEqual(B.reported, 2)

        # Now reconfigure the queue to be non-branched and run the same test
        # again.
        conf = textwrap.dedent(
            """
            - queue:
                name: {}
                per-branch: false
            """).format(queue_name)

        file_dict = {'zuul.d/queue.yaml': conf}
        C = self.fake_gerrit.addFakeChange(queue_repo, 'master', 'A',
                                           files=file_dict)
        C.setMerged()
        self.fake_gerrit.addEvent(C.getChangeMergedEvent())
        self.waitUntilSettled()

        self.executor_server.hold_jobs_in_build = True
        D = self.fake_gerrit.addFakeChange(project, 'master', 'D')
        E = self.fake_gerrit.addFakeChange(project, 'stable', 'E')
        D.addApproval('Code-Review', 2)
        E.addApproval('Code-Review', 2)

        self.executor_server.failJob('project-test', D)

        # Let first go A into gate then B
        self.fake_gerrit.addEvent(D.addApproval('Approved', 1))
        self.waitUntilSettled()

        self.fake_gerrit.addEvent(E.addApproval('Approved', 1))
        self.waitUntilSettled()

        # There should be two project-test jobs in a shared queue
        self.assertBuilds([
            dict(name='project-test', changes='4,1'),
            dict(name='project-test', changes='4,1 5,1'),
        ])
        tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
        _, p = tenant.getProject(project)
        q1 = tenant.layout.pipeline_managers['gate'].state.getQueue(
            p.canonical_name, 'master')
        q2 = tenant.layout.pipeline_managers['gate'].state.getQueue(
            p.canonical_name, 'stable')
        q3 = tenant.layout.pipeline_managers['gate'].state.getQueue(
            p.canonical_name, None)

        # There should be no branch specific queues anymore
        self.assertEqual(q1, None)
        self.assertEqual(q2, None)
        self.assertEqual(q3.name, queue_name)

        # Both queues must contain one item
        self.assertEqual(len(q3.queue), 2)

        # Release project-test of D to make history after test deterministic
        self.executor_server.release('project-test', change='4 1')
        self.waitUntilSettled()

        # Fail job on the change on master
        self.executor_server.hold_jobs_in_build = False
        self.executor_server.release()
        self.waitUntilSettled()

        self.assertNotEqual(D.data['status'], 'MERGED')
        self.assertEqual(E.data['status'], 'MERGED')
        self.assertEqual(D.reported, 2)
        self.assertEqual(E.reported, 2)

        self.assertHistory([
            # Independent runs because of per branch queues
            dict(name='project-test', result='FAILURE', changes='1,1'),
            dict(name='project-test', result='SUCCESS', changes='2,1'),

            # Same queue with gate reset because of 4,1
            dict(name='project-test', result='FAILURE', changes='4,1'),

            # Result can be anything depending on timing of the gate reset.
            dict(name='project-test', changes='4,1 5,1'),
            dict(name='project-test', result='SUCCESS', changes='5,1'),
        ], ordered=False)

    def test_dependent_queues_per_branch(self):
        """
        Test that change queues can be different for different branches.

        In this case the project contains zuul config so the branches are
        known upfront and the queues are pre-seeded.
        """
        self._test_dependent_queues_per_branch('org/project')
        self.assertReportedStat(
            'zuul.tenant.tenant-one.pipeline.gate.queue.'
            'integrated.branch.master.current_changes',
            value='1', kind='g')
        self.assertReportedStat(
            'zuul.tenant.tenant-one.pipeline.gate.queue.'
            'integrated.branch.master.window',
            value='20', kind='g')
        self.assertReportedStat(
            'zuul.tenant.tenant-one.pipeline.gate.queue.'
            'integrated.branch.master.resident_time', kind='ms')
        self.assertReportedStat(
            'zuul.tenant.tenant-one.pipeline.gate.queue.'
            'integrated.branch.master.total_changes', value='1',
            kind='c')

    def test_dependent_queues_per_branch_no_config(self):
        """
        Test that change queues can be different for different branches.

        In this case we create changes for two branches in a repo that
        doesn't contain zuul config so the queues are not pre-seeded
        in the gate pipeline.
        """
        self._test_dependent_queues_per_branch('org/project2')

    def test_dependent_queues_per_branch_untrusted(self):
        """
        Test that change queues can be different for different branches.

        In this case we create changes for two branches in an untrusted repo
        that defines its own queue.
        """
        self._test_dependent_queues_per_branch(
            'org/project3', queue_name='integrated-untrusted',
            queue_repo='org/project3')

    def test_dependent_queues_per_branch_project_queue(self):
        """
        Test that change queues can be different for different branches.

        In this case we create changes for two branches in a repo that
        references the queue on project level instead of pipeline level.
        """
        self._test_dependent_queues_per_branch('org/project4')

    def test_duplicate_definition_on_branches(self):
        project = 'org/project3'
        self.create_branch(project, 'stable')
        self.fake_gerrit.addEvent(
            self.fake_gerrit.getFakeBranchCreatedEvent(project, 'stable'))
        self.waitUntilSettled()
        tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
        self.assertEqual(
            len(tenant.layout.loading_errors), 1,
            "No error should have been accumulated")
        # This error is expected and unrelated to this test (the
        # ignored configuration is used by other tests in this class):
        self.assertIn('Queue integrated already defined',
                      tenant.layout.loading_errors[0].error)

        # At this point we've verified that we can have identical
        # queue definitions on multiple branches without conflict.
        # Next, let's try to change the queue def on one branch so it
        # doesn't match (flip the per-branch boolean):
        conf = textwrap.dedent(
            """
            - queue:
                name: integrated-untrusted
                per-branch: false
            """)

        file_dict = {'zuul.d/queue.yaml': conf}
        A = self.fake_gerrit.addFakeChange(project, 'stable', 'A',
                                           files=file_dict)
        A.addApproval('Code-Review', 2)
        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.waitUntilSettled()
        self.assertEqual(len(A.messages), 1)
        self.assertTrue(
            'Queue integrated-untrusted does not match '
            'existing definition in branch master' in A.messages[0])
        self.assertEqual(A.data['status'], 'NEW')


class TestJobUpdateBrokenConfig(ZuulTestCase):
    tenant_config_file = 'config/job-update-broken/main.yaml'

    def test_fix_check_without_running(self):
        "Test that we can fix a broken check pipeline (don't run the job)"
        in_repo_conf = textwrap.dedent(
            """
            - job:
                name: existing-files
                files:
                  - README.txt

            - project-template:
                name: files-template
                check:
                  jobs:
                    - existing-files
                    - noop
            """)

        # When the config is broken, we don't override any files
        # matchers since we don't have a valid basis.  Since this
        # doesn't update README.txt, nothing should run.
        file_dict = {'zuul.d/existing.yaml': in_repo_conf}
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
                                           files=file_dict)
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()
        self.assertHistory([])
        self.assertEqual(A.reported, 1)

    def test_fix_check_with_running(self):
        "Test that we can fix a broken check pipeline (do run the job)"
        in_repo_conf = textwrap.dedent(
            """
            - job:
                name: existing-files
                files:
                  - README.txt

            - project-template:
                name: files-template
                check:
                  jobs:
                    - existing-files
            """)

        # When the config is broken, we don't override any files
        # matchers since we don't have a valid basis.  Since this
        # does update README.txt, the job should run.
        file_dict = {'zuul.d/existing.yaml': in_repo_conf,
                     'README.txt': ''}
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
                                           files=file_dict)
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()
        self.assertHistory([
            dict(name='existing-files', result='SUCCESS', changes='1,1'),
        ])
        self.assertEqual(A.reported, 1)


class TestJobUpdateFileMatcher(ZuulTestCase):
    tenant_config_file = 'config/job-update/main.yaml'

    def test_matchers(self):
        "Test matchers work as expected with no change"
        file_dict = {'README.txt': ''}
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
                                           files=file_dict)
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        file_dict = {'something_else': ''}
        B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B',
                                           files=file_dict)
        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        self.assertHistory([
            dict(name='existing-files', result='SUCCESS', changes='1,1'),
            dict(name='existing-irr', result='SUCCESS', changes='2,1'),
        ])

    def test_job_update(self):
        "Test matchers are overridden with a config update"
        in_repo_conf = textwrap.dedent(
            """
            - job:
                name: existing-files
                tags: foo
            - job:
                name: existing-irr
                tags: foo
            """)

        file_dict = {'zuul.d/new.yaml': in_repo_conf}
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
                                           files=file_dict)
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        self.assertHistory([
            dict(name='existing-files', result='SUCCESS', changes='1,1'),
            dict(name='existing-irr', result='SUCCESS', changes='1,1'),
        ], ordered=False)

    def test_job_update_files(self):
        "Test that changes to file matchers themselves don't run jobs"
        # Normally we want to ignore file matchers and run jobs if the
        # job config changes, but if the only thing about the job
        # config that changes *is* the file matchers, then we don't
        # want to run it.
        in_repo_conf = textwrap.dedent(
            """
            - job:
                name: existing-files
                files: 'doesnotexist'
            - job:
                name: existing-irr
                irrelevant-files:
                  - README
                  - ^zuul.d/.*$
                  - newthing
            """)

        file_dict = {'zuul.d/new.yaml': in_repo_conf}
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
                                           files=file_dict)
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        self.assertHistory([])

    def test_job_dependencies_update(self):
        "Test that a dependent job will run when depending job updates"
        in_repo_conf = textwrap.dedent(
            """
            # Same
            - job:
                name: existing-files
                files:
                  - README.txt

            # Changed
            - job:
                name: existing-irr
                files:
                  - README.txt

            - project:
                name: org/project
                check:
                  jobs:
                    - existing-files
                    - existing-irr:
                        dependencies: [existing-files]
            """)

        file_dict = {'zuul.d/existing.yaml': in_repo_conf}
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
                                           files=file_dict)
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        self.assertHistory([
            dict(name='existing-files', result='SUCCESS', changes='1,1'),
            dict(name='existing-irr', result='SUCCESS', changes='1,1'),
        ], ordered=False)

    def test_new_job(self):
        "Test matchers are overridden when creating a new job"
        in_repo_conf = textwrap.dedent(
            """
            - job:
                name: new-files
                parent: existing-files

            - project:
                check:
                  jobs:
                    - new-files
            """)

        file_dict = {'zuul.d/new.yaml': in_repo_conf}
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
                                           files=file_dict)
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        self.assertHistory([
            dict(name='new-files', result='SUCCESS', changes='1,1'),
        ])

    def test_patch_series(self):
        "Test that we diff to the nearest layout in a patch series"
        in_repo_conf = textwrap.dedent(
            """
            - job:
                name: new-files1
                parent: existing-files

            - project:
                check:
                  jobs:
                    - new-files1
            """)

        file_dict = {'zuul.d/new1.yaml': in_repo_conf}
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
                                           files=file_dict)

        in_repo_conf = textwrap.dedent(
            """
            - job:
                name: new-files2
                parent: existing-files

            - project:
                check:
                  jobs:
                    - new-files2
            """)

        file_dict = {'zuul.d/new2.yaml': in_repo_conf}
        B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B',
                                           files=file_dict)
        B.setDependsOn(A, 1)
        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        self.assertHistory([
            dict(name='new-files2', result='SUCCESS', changes='1,1 2,1'),
        ])

    def test_disable_match(self):
        "Test matchers are not overridden if we say so"
        in_repo_conf = textwrap.dedent(
            """
            - job:
                name: new-files
                parent: existing-files
                match-on-config-updates: false

            - project:
                check:
                  jobs:
                    - new-files
            """)

        file_dict = {'zuul.d/new.yaml': in_repo_conf}
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
                                           files=file_dict)
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()
        self.assertHistory([])

    def test_include_vars_update(self):
        "Test matchers are overridden with an include-vars update"
        in_repo_conf = textwrap.dedent(
            """
            foo: baz
            """)
        file_dict = {'vars.yaml': in_repo_conf}
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
                                           files=file_dict)
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        self.assertHistory([
            dict(name='existing-files', result='SUCCESS', changes='1,1'),
            dict(name='existing-irr', result='SUCCESS', changes='1,1'),
        ], ordered=False)

    def test_playbook_update(self):
        "Test matchers are overridden with a playbook update"
        in_repo_conf = textwrap.dedent(
            """
            # Noop change
            - hosts: all
              tasks: []
            """)
        file_dict = {'playbook.yaml': in_repo_conf}
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
                                           files=file_dict)
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        self.assertHistory([
            dict(name='existing-files', result='SUCCESS', changes='1,1'),
            dict(name='existing-irr', result='SUCCESS', changes='1,1'),
        ], ordered=False)


class TestJobUpdateFileMatcherTransitive(ZuulTestCase):
    tenant_config_file = 'config/job-update-transitive/main.yaml'

    def test_job_update_transitive(self):
        # Test that a change to the configuration of job C in the
        # series C -> B -> A causes all 3 jobs to run.
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()
        # None of these jobs run because no files matched (and no job
        # was updated)
        self.assertHistory([])

        in_repo_conf = textwrap.dedent(
            """
            - job:
                name: job3
                vars: {foo: bar}
            """)

        file_dict = {'zuul.d/extend.yaml': in_repo_conf}
        B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B',
                                           files=file_dict)
        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        self.assertHistory([
            dict(name='job1', result='SUCCESS', changes='2,1'),
            dict(name='job2', result='SUCCESS', changes='2,1'),
            dict(name='job3', result='SUCCESS', changes='2,1'),
        ], ordered=False)

    def test_file_matcher_transitive(self):
        # This is really a file matcher test, but it's identical in
        # behavior to the above test so we do it here as well:
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()
        # None of these jobs run because no files matched (and no job
        # was updated)
        self.assertHistory([])

        file_dict = {'job3.txt': ''}
        B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B',
                                           files=file_dict)
        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        self.assertHistory([
            dict(name='job1', result='SUCCESS', changes='2,1'),
            dict(name='job2', result='SUCCESS', changes='2,1'),
            dict(name='job3', result='SUCCESS', changes='2,1'),
        ], ordered=False)

    def test_job_update_transitive2(self):
        # Similar to test_job_update_transitive but test:
        # job6 --(hard)--> job5 --(soft)--> job4
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()
        # None of these jobs run because no files matched (and no job
        # was updated)
        self.assertHistory([])

        in_repo_conf = textwrap.dedent(
            """
            - job:
                name: job6
                vars: {foo: bar}
            """)

        file_dict = {'zuul.d/extend.yaml': in_repo_conf}
        B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B',
                                           files=file_dict)
        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        self.assertHistory([
            dict(name='job5', result='SUCCESS', changes='2,1'),
            dict(name='job6', result='SUCCESS', changes='2,1'),
        ], ordered=False)

    def test_file_matcher_transitive2(self):
        # Similar to test_file_matcher_transitive, but test:
        # job6 --(hard)--> job5 --(soft)--> job4
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()
        # None of these jobs run because no files matched (and no job
        # was updated)
        self.assertHistory([])

        file_dict = {'job6.txt': ''}
        B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B',
                                           files=file_dict)
        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        self.assertHistory([
            dict(name='job5', result='SUCCESS', changes='2,1'),
            dict(name='job6', result='SUCCESS', changes='2,1'),
        ], ordered=False)

    def test_job_update_transitive3(self):
        # Similar to test_job_update_transitive but test:
        # job9 --(soft)--> job8 --(hard)--> job7
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()
        # None of these jobs run because no files matched (and no job
        # was updated)
        self.assertHistory([])

        in_repo_conf = textwrap.dedent(
            """
            - job:
                name: job9
                vars: {foo: bar}
            """)

        file_dict = {'zuul.d/extend.yaml': in_repo_conf}
        B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B',
                                           files=file_dict)
        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        self.assertHistory([
            dict(name='job9', result='SUCCESS', changes='2,1'),
        ], ordered=False)

    def test_file_matcher_transitive3(self):
        # Similar to test_file_matcher_transitive, but test:
        # job9 --(soft)--> job8 --(hard)--> job7
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()
        # None of these jobs run because no files matched (and no job
        # was updated)
        self.assertHistory([])

        file_dict = {'job9.txt': ''}
        B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B',
                                           files=file_dict)
        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        self.assertHistory([
            dict(name='job9', result='SUCCESS', changes='2,1'),
        ], ordered=False)


class TestExecutor(ZuulTestCase):
    tenant_config_file = 'config/single-tenant/main.yaml'

    def assertFinalState(self):
        # In this test, we expect to shut down in a non-final state,
        # so skip these checks.
        pass

    def cleanupTestServers(self):
        pass

    def assertCleanShutdown(self):
        self.log.debug("Assert clean shutdown")

        # After shutdown, make sure no jobs are running
        self.assertEqual({}, self.executor_server.job_workers)
        super().cleanupTestServers()

        # Make sure that git.Repo objects have been garbage collected.
        gc.disable()
        try:
            gc.collect()
            for obj in gc.get_objects():
                if isinstance(obj, git.Repo):
                    self.log.debug("Leaked git repo object: %s" % repr(obj))
            gc.enable()
        finally:
            gc.enable()

    def test_executor_shutdown(self):
        "Test that the executor can shut down with jobs running"

        self.executor_server.hold_jobs_in_build = True
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        A.addApproval('Code-Review', 2)
        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.waitUntilSettled()


class TestDependencyGraph(ZuulTestCase):
    tenant_config_file = 'config/dependency-graph/main.yaml'

    def test_dependeny_graph_dispatch_jobs_once(self):
        "Test a job in a dependency graph is queued only once"
        # Job dependencies, starting with A
        #     A
        #    / \
        #   B   C
        #  / \ / \
        # D   F   E
        #     |
        #     G

        self.executor_server.hold_jobs_in_build = True
        change = self.fake_gerrit.addFakeChange(
            'org/project', 'master', 'change')
        change.addApproval('Code-Review', 2)
        self.fake_gerrit.addEvent(change.addApproval('Approved', 1))

        self.waitUntilSettled()
        self.assertEqual([b.name for b in self.builds], ['A'])

        self.executor_server.release('A')
        self.waitUntilSettled()
        self.assertEqual(sorted(b.name for b in self.builds), ['B', 'C'])

        self.executor_server.release('B')
        self.waitUntilSettled()
        self.assertEqual(sorted(b.name for b in self.builds), ['C', 'D'])

        self.executor_server.release('D')
        self.waitUntilSettled()
        self.assertEqual([b.name for b in self.builds], ['C'])

        self.executor_server.release('C')
        self.waitUntilSettled()
        self.assertEqual(sorted(b.name for b in self.builds), ['E', 'F'])

        self.executor_server.release('F')
        self.waitUntilSettled()
        self.assertEqual(sorted(b.name for b in self.builds), ['E', 'G'])

        self.executor_server.release('G')
        self.waitUntilSettled()
        self.assertEqual([b.name for b in self.builds], ['E'])

        self.executor_server.release('E')
        self.waitUntilSettled()
        self.assertEqual(len(self.builds), 0)

        self.executor_server.hold_jobs_in_build = False
        self.executor_server.release()
        self.waitUntilSettled()

        self.assertEqual(len(self.builds), 0)
        self.assertEqual(len(self.history), 7)

        self.assertEqual(change.data['status'], 'MERGED')
        self.assertEqual(change.reported, 2)

    def test_jobs_launched_only_if_all_dependencies_are_successful(self):
        "Test that a job waits till all dependencies are successful"
        # Job dependencies, starting with A
        #     A
        #    / \
        #   B   C*
        #  / \ / \
        # D   F   E
        #     |
        #     G

        self.executor_server.hold_jobs_in_build = True
        change = self.fake_gerrit.addFakeChange(
            'org/project', 'master', 'change')
        change.addApproval('Code-Review', 2)

        self.executor_server.failJob('C', change)

        self.fake_gerrit.addEvent(change.addApproval('Approved', 1))

        self.waitUntilSettled()
        self.assertEqual([b.name for b in self.builds], ['A'])

        self.executor_server.release('A')
        self.waitUntilSettled()
        self.assertEqual(sorted(b.name for b in self.builds), ['B', 'C'])

        self.executor_server.release('B')
        self.waitUntilSettled()
        self.assertEqual(sorted(b.name for b in self.builds), ['C', 'D'])

        self.executor_server.release('D')
        self.waitUntilSettled()
        self.assertEqual([b.name for b in self.builds], ['C'])

        self.executor_server.release('C')
        self.waitUntilSettled()
        self.assertEqual(len(self.builds), 0)

        self.executor_server.hold_jobs_in_build = False
        self.executor_server.release()
        self.waitUntilSettled()

        self.assertEqual(len(self.builds), 0)
        self.assertEqual(len(self.history), 4)

        self.assertEqual(change.data['status'], 'NEW')
        self.assertEqual(change.reported, 2)

    @simple_layout('layouts/soft-dependencies-error.yaml')
    def test_soft_dependencies_error(self):
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()
        self.assertHistory([])
        self.assertEqual(len(A.messages), 1)
        self.assertTrue('Job project-merge not defined' in A.messages[0])
        self.log.info(A.messages)

    @simple_layout('layouts/soft-dependencies.yaml')
    def test_soft_dependencies(self):
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()
        self.assertHistory([
            dict(name='deploy', result='SUCCESS', changes='1,1'),
        ], ordered=False)

    @simple_layout('layouts/not-skip-when-reenqueue.yaml')
    def test_child_with_soft_dependency_should_not_skip(self):
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        self.executor_server.hold_jobs_in_build = True
        self.executor_server.returnData(
            'grand-parent', A,
            {'zuul':
                {'child_jobs': ['parent2']}
            }
        )

        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        self.executor_server.release('grand-parent')
        self.waitUntilSettled()
        # grand-parent success, parent1 skipped, parent2 running
        self.assertHistory([
            dict(name='grand-parent', result='SUCCESS', changes='1,1'),
        ], ordered=False)
        self.assertBuilds([dict(name='parent2')])

        # Reconfigure to trigger a re-enqueue, this should not cause job
        # 'child' to be skipped because parent1 was skipped
        self.scheds.execute(lambda app: app.sched.reconfigure(app.config))

        self.executor_server.release('parent1')
        self.executor_server.release('parent2')
        self.waitUntilSettled()
        # grand-parent success, parent1 skipped, parent2 success, child running
        self.assertHistory([
            dict(name='grand-parent', result='SUCCESS', changes='1,1'),
            dict(name='parent2', result='SUCCESS', changes='1,1'),
        ], ordered=False)
        self.assertBuilds([dict(name='child')])

        self.executor_server.release('child')
        self.waitUntilSettled()
        # grand-parent success, parent1 skipped, parent2 success, child success
        self.assertHistory([
            dict(name='grand-parent', result='SUCCESS', changes='1,1'),
            dict(name='parent2', result='SUCCESS', changes='1,1'),
            dict(name='child', result='SUCCESS', changes='1,1'),
        ], ordered=False)
        self.assertBuilds([])

    @simple_layout('layouts/soft-dependencies.yaml')
    def test_soft_dependencies_failure(self):
        file_dict = {'main.c': 'test'}
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
                                           files=file_dict)
        self.executor_server.failJob('build', A)
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()
        self.assertHistory([
            dict(name='build', result='FAILURE', changes='1,1'),
        ], ordered=False)
        self.assertIn('Skipped due to failed job build', A.messages[0])


class TestDuplicatePipeline(ZuulTestCase):
    tenant_config_file = 'config/duplicate-pipeline/main.yaml'

    def test_duplicate_pipelines(self):
        "Test that a change matching multiple pipelines works"

        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        self.fake_gerrit.addEvent(A.getChangeRestoredEvent())
        self.waitUntilSettled()

        self.assertHistory([
            dict(name='project-test1', result='SUCCESS', changes='1,1',
                 pipeline='dup1'),
            dict(name='project-test1', result='SUCCESS', changes='1,1',
                 pipeline='dup2'),
        ], ordered=False)

        self.assertEqual(len(A.messages), 2)

        if 'dup1' in A.messages[0]:
            self.assertIn('dup1', A.messages[0])
            self.assertNotIn('dup2', A.messages[0])
            self.assertIn('project-test1', A.messages[0])
            self.assertIn('dup2', A.messages[1])
            self.assertNotIn('dup1', A.messages[1])
            self.assertIn('project-test1', A.messages[1])
        else:
            self.assertIn('dup1', A.messages[1])
            self.assertNotIn('dup2', A.messages[1])
            self.assertIn('project-test1', A.messages[1])
            self.assertIn('dup2', A.messages[0])
            self.assertNotIn('dup1', A.messages[0])
            self.assertIn('project-test1', A.messages[0])


class TestSchedulerRegexProject(ZuulTestCase):
    tenant_config_file = 'config/regex-project/main.yaml'

    def test_regex_project(self):
        "Test that changes are tested in parallel and merged in series"

        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
        C = self.fake_gerrit.addFakeChange('org/project2', 'master', 'C')

        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
        self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))

        self.waitUntilSettled()

        # We expect the following builds:
        #  - 1 for org/project
        #  - 3 for org/project1
        #  - 3 for org/project2
        self.assertEqual(len(self.history), 7)
        self.assertEqual(A.reported, 1)
        self.assertEqual(B.reported, 1)
        self.assertEqual(C.reported, 1)

        self.assertHistory([
            dict(name='project-test', result='SUCCESS', changes='1,1'),
            dict(name='project-test1', result='SUCCESS', changes='2,1'),
            dict(name='project-common-test', result='SUCCESS', changes='2,1'),
            dict(name='project-common-test-canonical', result='SUCCESS',
                 changes='2,1'),
            dict(name='project-test2', result='SUCCESS', changes='3,1'),
            dict(name='project-common-test', result='SUCCESS', changes='3,1'),
            dict(name='project-common-test-canonical', result='SUCCESS',
                 changes='3,1'),
        ], ordered=False)


class TestSchedulerTemplatedProject(ZuulTestCase):
    tenant_config_file = 'config/templated-project/main.yaml'

    def test_job_from_templates_executed(self):
        "Test whether a job generated via a template can be executed"

        A = self.fake_gerrit.addFakeChange(
            'org/templated-project', 'master', 'A')
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        self.assertEqual(self.getJobFromHistory('project-test1').result,
                         'SUCCESS')
        self.assertEqual(self.getJobFromHistory('project-test2').result,
                         'SUCCESS')

    def test_layered_templates(self):
        "Test whether a job generated via a template can be executed"

        A = self.fake_gerrit.addFakeChange(
            'org/layered-project', 'master', 'A')
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        self.assertEqual(self.getJobFromHistory('project-test1').result,
                         'SUCCESS')
        self.assertEqual(self.getJobFromHistory('project-test2').result,
                         'SUCCESS')
        self.assertEqual(self.getJobFromHistory('layered-project-test3'
                                                ).result, 'SUCCESS')
        self.assertEqual(self.getJobFromHistory('layered-project-test4'
                                                ).result, 'SUCCESS')
        self.assertEqual(self.getJobFromHistory('layered-project-foo-test5'
                                                ).result, 'SUCCESS')
        self.assertEqual(self.getJobFromHistory('project-test6').result,
                         'SUCCESS')

    def test_unimplied_branch_matchers(self):
        # This tests that there are no implied branch matchers added
        # to project templates in unbranched projects.
        self.create_branch('org/layered-project', 'stable')
        self.fake_gerrit.addEvent(
            self.fake_gerrit.getFakeBranchCreatedEvent(
                'org/layered-project', 'stable'))
        self.waitUntilSettled()

        A = self.fake_gerrit.addFakeChange(
            'org/layered-project', 'stable', 'A')

        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        self.assertEqual(self.getJobFromHistory('project-test1').result,
                         'SUCCESS')
        self.log.info(
            self.getJobFromHistory('project-test1').
            parameters['zuul']['_inheritance_path'])

    def test_implied_branch_matchers(self):
        # This tests that there is an implied branch matcher when a
        # template is used on an in-repo project pipeline definition.
        self.create_branch('untrusted-config', 'stable')
        self.fake_gerrit.addEvent(
            self.fake_gerrit.getFakeBranchCreatedEvent(
                'untrusted-config', 'stable'))
        self.waitUntilSettled()

        A = self.fake_gerrit.addFakeChange(
            'untrusted-config', 'stable', 'A')

        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        self.assertEqual(self.getJobFromHistory('project-test1').result,
                         'SUCCESS')
        self.log.info(
            self.getJobFromHistory('project-test1').
            parameters['zuul']['_inheritance_path'])

        # Now create a new branch named stable-foo and change the project
        # pipeline
        self.create_branch('untrusted-config', 'stable-foo')
        self.fake_gerrit.addEvent(
            self.fake_gerrit.getFakeBranchCreatedEvent(
                'untrusted-config', 'stable-foo'))
        self.waitUntilSettled()

        in_repo_conf = textwrap.dedent(
            """
            - project:
                name: untrusted-config
                templates:
                  - test-three-and-four
                check:
                  jobs:
                    - project-test7
            """)
        file_dict = {'zuul.d/project.yaml': in_repo_conf}
        B = self.fake_gerrit.addFakeChange('untrusted-config', 'stable-foo',
                                           'B', files=file_dict)
        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        self.assertHistory([
            dict(name='project-test1', result='SUCCESS', changes='1,1'),
            dict(name='project-test2', result='SUCCESS', changes='1,1'),
            dict(name='project-test7', result='SUCCESS', changes='2,1'),
            dict(name='layered-project-test3', result='SUCCESS',
                 changes='2,1'),
            dict(name='layered-project-test4', result='SUCCESS',
                 changes='2,1'),
        ], ordered=False)

        # Inheritance path should not contain items from branch stable
        # This tests that not only is it the case that the stable
        # branch project-template did not apply, but also that the
        # stable branch definitions of the project-test7 did not apply
        # (since the job definitions also have implied branch
        # matchers).
        job = self.getJobFromHistory('project-test7', branch='stable-foo')
        inheritance_path = job.parameters['zuul']['_inheritance_path']
        self.assertEqual(len(inheritance_path), 4)
        stable_items = [x for x in inheritance_path
                        if 'untrusted-config/zuul.d/jobs.yaml@stable#' in x]
        self.assertEqual(len(stable_items), 0)


class TestSchedulerMerges(ZuulTestCase):
    tenant_config_file = 'config/merges/main.yaml'

    def _test_project_merge_mode(self, mode):
        self.executor_server.keep_jobdir = False
        project = 'org/project-%s' % mode
        self.executor_server.hold_jobs_in_build = True
        A = self.fake_gerrit.addFakeChange(project, 'master', 'A')
        B = self.fake_gerrit.addFakeChange(project, 'master', 'B')
        C = self.fake_gerrit.addFakeChange(project, 'master', 'C')
        if mode == 'cherry-pick':
            A.cherry_pick = True
            B.cherry_pick = True
            C.cherry_pick = True
        A.addApproval('Code-Review', 2)
        B.addApproval('Code-Review', 2)
        C.addApproval('Code-Review', 2)
        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
        self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
        self.waitUntilSettled()

        build = self.builds[-1]
        path = os.path.join(build.jobdir.src_root, 'review.example.com',
                            project)
        repo = git.Repo(path)
        repo_messages = [c.message.strip() for c in repo.iter_commits()]
        repo_messages.reverse()

        self.executor_server.hold_jobs_in_build = False
        self.executor_server.release()
        self.waitUntilSettled()

        return repo_messages

    def _test_merge(self, mode):
        expected_messages = [
            'initial commit',
            'add content from fixture',
            # the intermediate commits order is nondeterministic
            "Merge 'refs/changes/02/2/1'",
            "Merge 'refs/changes/03/3/1'",
        ]
        result = self._test_project_merge_mode(mode)
        self.assertEqual(result[:2], expected_messages[:2])
        self.assertEqual(result[-2:], expected_messages[-2:])

    def test_project_merge_mode_merge(self):
        self._test_merge('merge')

    def test_project_merge_mode_merge_resolve(self):
        self._test_merge('merge-resolve')

    def test_project_merge_mode_cherrypick(self):
        expected_messages = [
            'initial commit',
            'add content from fixture',
            'A-1',
            'B-1',
            'C-1']
        result = self._test_project_merge_mode('cherry-pick')
        self.assertEqual(result, expected_messages)

    def test_project_merge_mode_cherrypick_redundant(self):
        # A redundant commit (that is, one that has already been applied to the
        # working tree) should be skipped
        self.executor_server.keep_jobdir = False
        project = 'org/project-cherry-pick'
        files = {
            "foo.txt": "ABC",
        }
        A = self.fake_gerrit.addFakeChange(project, 'master', 'A', files=files)
        A.addApproval('Code-Review', 2)
        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.waitUntilSettled()

        self.executor_server.hold_jobs_in_build = True
        B = self.fake_gerrit.addFakeChange(project, 'master', 'B', files=files)
        B.addApproval('Code-Review', 2)
        self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
        self.waitUntilSettled()

        build = self.builds[-1]
        path = os.path.join(build.jobdir.src_root, 'review.example.com',
                            project)
        repo = git.Repo(path)
        repo_messages = [c.message.strip() for c in repo.iter_commits()]
        repo_messages.reverse()

        self.executor_server.hold_jobs_in_build = False
        self.executor_server.release()
        self.waitUntilSettled()

        expected_messages = [
            'initial commit',
            'add content from fixture',
            'A-1',
        ]
        self.assertHistory([
            dict(name='project-test1', result='SUCCESS', changes='1,1'),
            dict(name='project-test1', result='SUCCESS', changes='2,1'),
        ])
        self.assertEqual(A.data['status'], 'MERGED')
        self.assertEqual(B.data['status'], 'MERGED')
        self.assertEqual(repo_messages, expected_messages)

    def test_project_merge_mode_cherrypick_empty(self):
        # An empty commit (that is, one that doesn't modify any files) should
        # be preserved
        self.executor_server.keep_jobdir = False
        project = 'org/project-cherry-pick'
        self.executor_server.hold_jobs_in_build = True
        A = self.fake_gerrit.addFakeChange(project, 'master', 'A', empty=True)
        A.addApproval('Code-Review', 2)
        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.waitUntilSettled()

        build = self.builds[-1]
        path = os.path.join(build.jobdir.src_root, 'review.example.com',
                            project)
        repo = git.Repo(path)
        repo_messages = [c.message.strip() for c in repo.iter_commits()]
        repo_messages.reverse()

        changed_files = list(repo.commit("HEAD").diff(repo.commit("HEAD~1")))
        self.assertEqual(changed_files, [])

        self.executor_server.hold_jobs_in_build = False
        self.executor_server.release()
        self.waitUntilSettled()

        expected_messages = [
            'initial commit',
            'add content from fixture',
            'A-1',
        ]
        self.assertHistory([
            dict(name='project-test1', result='SUCCESS', changes='1,1'),
        ])
        self.assertEqual(A.data['status'], 'MERGED')
        self.assertEqual(repo_messages, expected_messages)

    def test_project_merge_mode_cherrypick_branch_merge(self):
        "Test that branches can be merged together in cherry-pick mode"
        self.create_branch('org/project-merge-branches', 'mp')
        self.fake_gerrit.addEvent(
            self.fake_gerrit.getFakeBranchCreatedEvent(
                'org/project-merge-branches', 'mp'))
        self.waitUntilSettled()

        path = os.path.join(self.upstream_root, 'org/project-merge-branches')
        repo = git.Repo(path)
        master_sha = repo.heads.master.commit.hexsha
        mp_sha = repo.heads.mp.commit.hexsha

        self.executor_server.hold_jobs_in_build = True
        M = self.fake_gerrit.addFakeChange(
            'org/project-merge-branches', 'master', 'M',
            merge_parents=[
                master_sha,
                mp_sha,
            ])
        M.addApproval('Code-Review', 2)
        self.fake_gerrit.addEvent(M.addApproval('Approved', 1))
        self.waitUntilSettled()

        self.executor_server.release('.*-merge')
        self.waitUntilSettled()

        build = self.builds[-1]
        self.assertEqual(build.parameters['zuul']['branch'], 'master')
        path = os.path.join(build.jobdir.src_root, 'review.example.com',
                            "org/project-merge-branches")
        repo = git.Repo(path)
        repo_messages = [c.message.strip() for c in repo.iter_commits()]
        repo_messages.reverse()
        correct_messages = [
            'initial commit',
            'add content from fixture',
            'mp commit',
            'M-1']
        self.assertEqual(repo_messages, correct_messages)

        self.executor_server.hold_jobs_in_build = False
        self.executor_server.release()
        self.waitUntilSettled()

    def test_merge_branch(self):
        "Test that the right commits are on alternate branches"
        self.create_branch('org/project-merge-branches', 'mp')
        self.fake_gerrit.addEvent(
            self.fake_gerrit.getFakeBranchCreatedEvent(
                'org/project-merge-branches', 'mp'))
        self.waitUntilSettled()

        self.executor_server.hold_jobs_in_build = True
        A = self.fake_gerrit.addFakeChange(
            'org/project-merge-branches', 'mp', 'A')
        B = self.fake_gerrit.addFakeChange(
            'org/project-merge-branches', 'mp', 'B')
        C = self.fake_gerrit.addFakeChange(
            'org/project-merge-branches', 'mp', 'C')
        A.addApproval('Code-Review', 2)
        B.addApproval('Code-Review', 2)
        C.addApproval('Code-Review', 2)
        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
        self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
        self.waitUntilSettled()

        self.executor_server.release('.*-merge')
        self.waitUntilSettled()
        self.executor_server.release('.*-merge')
        self.waitUntilSettled()
        self.executor_server.release('.*-merge')
        self.waitUntilSettled()

        build = self.builds[-1]
        self.assertEqual(build.parameters['zuul']['branch'], 'mp')
        path = os.path.join(build.jobdir.src_root, 'review.example.com',
                            'org/project-merge-branches')
        repo = git.Repo(path)

        repo_messages = [c.message.strip() for c in repo.iter_commits()]
        repo_messages.reverse()
        correct_messages = [
            'initial commit',
            'add content from fixture',
            'mp commit',
            'A-1', 'B-1', 'C-1']
        self.assertEqual(repo_messages, correct_messages)

        self.executor_server.hold_jobs_in_build = False
        self.executor_server.release()
        self.waitUntilSettled()

    def test_merge_multi_branch(self):
        "Test that dependent changes on multiple branches are merged"
        self.create_branch('org/project-merge-branches', 'mp')
        self.fake_gerrit.addEvent(
            self.fake_gerrit.getFakeBranchCreatedEvent(
                'org/project-merge-branches', 'mp'))
        self.waitUntilSettled()

        self.executor_server.hold_jobs_in_build = True
        A = self.fake_gerrit.addFakeChange(
            'org/project-merge-branches', 'master', 'A')
        B = self.fake_gerrit.addFakeChange(
            'org/project-merge-branches', 'mp', 'B')
        C = self.fake_gerrit.addFakeChange(
            'org/project-merge-branches', 'master', 'C')
        A.addApproval('Code-Review', 2)
        B.addApproval('Code-Review', 2)
        C.addApproval('Code-Review', 2)
        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
        self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
        self.waitUntilSettled()

        job_A = None
        for job in self.builds:
            if 'project-merge' in job.name:
                job_A = job

        path = os.path.join(job_A.jobdir.src_root, 'review.example.com',
                            'org/project-merge-branches')
        repo = git.Repo(path)
        repo_messages = [c.message.strip()
                         for c in repo.iter_commits()]
        repo_messages.reverse()
        correct_messages = [
            'initial commit', 'add content from fixture', 'A-1']
        self.assertEqual(repo_messages, correct_messages)

        self.executor_server.release('.*-merge')
        self.waitUntilSettled()

        job_B = None
        for job in self.builds:
            if 'project-merge' in job.name:
                job_B = job

        path = os.path.join(job_B.jobdir.src_root, 'review.example.com',
                            'org/project-merge-branches')
        repo = git.Repo(path)
        repo_messages = [c.message.strip() for c in repo.iter_commits()]
        repo_messages.reverse()
        correct_messages = [
            'initial commit', 'add content from fixture', 'mp commit', 'B-1']
        self.assertEqual(repo_messages, correct_messages)

        self.executor_server.release('.*-merge')
        self.waitUntilSettled()

        job_C = None
        for job in self.builds:
            if 'project-merge' in job.name:
                job_C = job

        path = os.path.join(job_C.jobdir.src_root, 'review.example.com',
                            'org/project-merge-branches')
        repo = git.Repo(path)
        repo_messages = [c.message.strip() for c in repo.iter_commits()]

        repo_messages.reverse()
        correct_messages = [
            'initial commit', 'add content from fixture',
            'A-1', 'C-1']
        # Ensure the right commits are in the history for this ref
        self.assertEqual(repo_messages, correct_messages)

        self.executor_server.hold_jobs_in_build = False
        self.executor_server.release()
        self.waitUntilSettled()


class TestSemaphore(ZuulTestCase):
    tenant_config_file = 'config/semaphore/main.yaml'

    def test_semaphore_one(self):
        "Test semaphores with max=1 (mutex)"
        tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')

        self.executor_server.hold_jobs_in_build = True

        # Pause nodepool so we can check the ordering of getting the nodes
        # and aquiring the semaphore.
        self.fake_nodepool.paused = True

        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
        self.assertEqual(
            len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")),
            0)

        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        status = tenant.layout.pipeline_managers["check"].formatStatusJSON()
        jobs = status["change_queues"][0]["heads"][0][0]["jobs"]
        self.assertEqual(jobs[0]["waiting_status"],
                         'node request: 200-0000000000')
        self.assertEqual(jobs[1]["waiting_status"],
                         'node request: 200-0000000001')
        self.assertEqual(jobs[2]["waiting_status"],
                         'semaphores: test-semaphore')

        # By default we first lock the semaphore and then get the nodes
        # so at this point the semaphore needs to be aquired.
        self.assertEqual(
            len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")),
            1)
        self.fake_nodepool.paused = False
        self.waitUntilSettled()

        self.assertEqual(len(self.builds), 3)
        self.assertEqual(self.builds[0].name, 'project-test1')
        self.assertEqual(self.builds[1].name, 'semaphore-one-test1')
        self.assertEqual(self.builds[2].name, 'project-test1')

        self.executor_server.release('semaphore-one-test1')
        self.waitUntilSettled()

        self.assertEqual(len(self.builds), 3)
        self.assertEqual(self.builds[0].name, 'project-test1')
        self.assertEqual(self.builds[1].name, 'project-test1')
        self.assertEqual(self.builds[2].name, 'semaphore-one-test2')
        self.assertEqual(
            len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")),
            1)

        self.executor_server.release('semaphore-one-test2')
        self.waitUntilSettled()

        self.assertEqual(len(self.builds), 3)
        self.assertEqual(self.builds[0].name, 'project-test1')
        self.assertEqual(self.builds[1].name, 'project-test1')
        self.assertEqual(self.builds[2].name, 'semaphore-one-test1')
        self.assertEqual(
            len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")),
            1)

        self.executor_server.release('semaphore-one-test1')
        self.waitUntilSettled()

        self.assertEqual(len(self.builds), 3)
        self.assertEqual(self.builds[0].name, 'project-test1')
        self.assertEqual(self.builds[1].name, 'project-test1')
        self.assertEqual(self.builds[2].name, 'semaphore-one-test2')
        self.assertEqual(
            len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")),
            1)

        self.executor_server.release('semaphore-one-test2')
        self.waitUntilSettled()

        self.assertEqual(len(self.builds), 2)
        self.assertEqual(self.builds[0].name, 'project-test1')
        self.assertEqual(self.builds[1].name, 'project-test1')
        self.assertEqual(
            len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")),
            0)

        self.executor_server.hold_jobs_in_build = False
        self.executor_server.release()

        self.waitUntilSettled()
        self.assertEqual(len(self.builds), 0)

        self.assertEqual(A.reported, 1)
        self.assertEqual(B.reported, 1)
        self.assertEqual(
            len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")),
            0)

        self.assertReportedStat(
            'zuul.tenant.tenant-one.semaphore.test-semaphore.holders',
            value='1', kind='g')
        self.assertReportedStat(
            'zuul.tenant.tenant-one.semaphore.test-semaphore.holders',
            value='0', kind='g')

    def test_semaphore_two(self):
        "Test semaphores with max>1"
        tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')

        self.executor_server.hold_jobs_in_build = True
        A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
        B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
        self.assertEqual(
            len(tenant.semaphore_handler.semaphoreHolders(
                "test-semaphore-two")), 0)

        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        self.assertEqual(len(self.builds), 4)
        self.assertEqual(self.builds[0].name, 'project-test1')
        self.assertEqual(self.builds[1].name, 'semaphore-two-test1')
        self.assertEqual(self.builds[2].name, 'semaphore-two-test2')
        self.assertEqual(self.builds[3].name, 'project-test1')
        self.assertEqual(
            len(tenant.semaphore_handler.semaphoreHolders(
                "test-semaphore-two")), 2)

        self.executor_server.release('semaphore-two-test1')
        self.waitUntilSettled()

        self.assertEqual(len(self.builds), 4)
        self.assertEqual(self.builds[0].name, 'project-test1')
        self.assertEqual(self.builds[1].name, 'semaphore-two-test2')
        self.assertEqual(self.builds[2].name, 'project-test1')
        self.assertEqual(self.builds[3].name, 'semaphore-two-test1')
        self.assertEqual(
            len(tenant.semaphore_handler.semaphoreHolders(
                "test-semaphore-two")), 2)

        self.executor_server.release('semaphore-two-test2')
        self.waitUntilSettled()

        self.assertEqual(len(self.builds), 4)
        self.assertEqual(self.builds[0].name, 'project-test1')
        self.assertEqual(self.builds[1].name, 'project-test1')
        self.assertEqual(self.builds[2].name, 'semaphore-two-test1')
        self.assertEqual(self.builds[3].name, 'semaphore-two-test2')
        self.assertEqual(
            len(tenant.semaphore_handler.semaphoreHolders(
                "test-semaphore-two")), 2)

        self.executor_server.release('semaphore-two-test1')
        self.waitUntilSettled()

        self.assertEqual(len(self.builds), 3)
        self.assertEqual(self.builds[0].name, 'project-test1')
        self.assertEqual(self.builds[1].name, 'project-test1')
        self.assertEqual(self.builds[2].name, 'semaphore-two-test2')
        self.assertEqual(
            len(tenant.semaphore_handler.semaphoreHolders(
                "test-semaphore-two")), 1)

        self.executor_server.release('semaphore-two-test2')
        self.waitUntilSettled()

        self.assertEqual(len(self.builds), 2)
        self.assertEqual(self.builds[0].name, 'project-test1')
        self.assertEqual(self.builds[1].name, 'project-test1')
        self.assertEqual(
            len(tenant.semaphore_handler.semaphoreHolders(
                "test-semaphore-two")), 0)

        self.executor_server.hold_jobs_in_build = False
        self.executor_server.release()

        self.waitUntilSettled()
        self.assertEqual(len(self.builds), 0)

        self.assertEqual(A.reported, 1)
        self.assertEqual(B.reported, 1)

    def test_semaphore_node_failure(self):
        "Test semaphore and node failure"
        tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')

        # Pause nodepool so we can fail the node request later
        self.fake_nodepool.pause()

        A = self.fake_gerrit.addFakeChange('org/project2', 'master', 'A')
        self.assertEqual(
            len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")),
            0)
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        # By default we first lock the semaphore and then get the nodes
        # so at this point the semaphore needs to be aquired.
        self.assertEqual(
            len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")),
            1)

        # Fail the node request and unpause
        req = self.fake_nodepool.getNodeRequests()[0]
        self.fake_nodepool.addFailRequest(req)
        self.fake_nodepool.unpause()
        self.waitUntilSettled()

        # At this point the job that holds the semaphore failed with
        # node_failure and the semaphore must be released.
        self.assertEqual(
            len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")),
            0)

        self.assertEqual(1, A.reported)
        self.assertTrue(re.search('semaphore-one-test3 .* NODE_FAILURE',
                                  A.messages[0]))

    def test_semaphore_resources_first(self):
        "Test semaphores with max=1 (mutex) and get resources first"
        tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')

        self.executor_server.hold_jobs_in_build = True

        # Pause nodepool so we can check the ordering of getting the nodes
        # and aquiring the semaphore.
        self.fake_nodepool.paused = True

        A = self.fake_gerrit.addFakeChange('org/project3', 'master', 'A')
        B = self.fake_gerrit.addFakeChange('org/project3', 'master', 'B')
        self.assertEqual(
            len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")),
            0)

        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        # Here we first get the resources and then lock the semaphore
        # so at this point the semaphore should not be aquired.
        self.assertEqual(
            len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")),
            0)
        self.fake_nodepool.paused = False
        self.waitUntilSettled()

        self.assertEqual(len(self.builds), 3)
        self.assertEqual(self.builds[0].name, 'project-test1')
        self.assertEqual(self.builds[1].name,
                         'semaphore-one-test1-resources-first')
        self.assertEqual(self.builds[2].name, 'project-test1')

        self.executor_server.release('semaphore-one-test1')
        self.waitUntilSettled()
        self.assertEqual(len(self.builds), 3)
        self.assertEqual(self.builds[0].name, 'project-test1')
        self.assertEqual(self.builds[1].name, 'project-test1')
        self.assertEqual(self.builds[2].name,
                         'semaphore-one-test2-resources-first')
        self.assertEqual(
            len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")),
            1)

        self.executor_server.hold_jobs_in_build = False
        self.executor_server.release()
        self.waitUntilSettled()

    def test_semaphore_resources_first_node_failure(self):
        "Test semaphore and node failure"
        tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')

        # Pause nodepool so we can fail the node request later
        self.fake_nodepool.pause()

        A = self.fake_gerrit.addFakeChange('org/project4', 'master', 'A')
        self.assertEqual(
            len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")),
            0)
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        # With resources first we first get the nodes so at this point the
        # semaphore must not be aquired.
        self.assertEqual(len(tenant.semaphore_handler.semaphoreHolders(
            "test-semaphore")), 0)

        # Fail the node request and unpause
        req = self.fake_nodepool.getNodeRequests()[0]
        self.fake_nodepool.addFailRequest(req)
        self.fake_nodepool.unpause()
        self.waitUntilSettled()

        # At this point the job should never have acuired a semaphore so check
        # that it still has not locked a semaphore.
        self.assertEqual(len(tenant.semaphore_handler.semaphoreHolders(
            "test-semaphore")), 0)
        self.assertEqual(1, A.reported)
        self.assertTrue(
            re.search('semaphore-one-test1-resources-first .* NODE_FAILURE',
                      A.messages[0]))

    def test_semaphore_zk_error(self):
        "Test semaphore release with zk error"
        tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')

        A = self.fake_gerrit.addFakeChange('org/project2', 'master', 'A')
        self.assertEqual(
            len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")),
            0)

        # Simulate a single zk error in useNodeSet
        orig_useNodeSet = self.scheds.first.sched.nodepool.useNodeSet

        def broken_use_nodeset(nodeset, tenant_name, project_name):
            # restore original useNodeSet
            self.scheds.first.sched.nodepool.useNodeSet = orig_useNodeSet
            raise NoNodeError()

        self.scheds.first.sched.nodepool.useNodeSet = broken_use_nodeset

        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        # The semaphore should be released
        self.assertEqual(
            len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")),
            0)

        # cleanup the queue
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

    def test_semaphore_abandon(self):
        "Test abandon with job semaphores"
        self.executor_server.hold_jobs_in_build = True
        tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')

        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        self.assertEqual(
            len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")),
            0)

        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        self.assertEqual(
            len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")),
            1)

        self.fake_gerrit.addEvent(A.getChangeAbandonedEvent())
        self.waitUntilSettled()

        # The check pipeline should be empty
        items = self.getAllItems('tenant-one', 'check')
        self.assertEqual(len(items), 0)

        # The semaphore should be released
        self.assertEqual(
            len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")),
            0)

        self.executor_server.hold_jobs_in_build = False
        self.executor_server.release()
        self.waitUntilSettled()

    def test_semaphore_abandon_pending_node_request(self):
        "Test abandon with job semaphores and pending node request"
        self.executor_server.hold_jobs_in_build = True

        # Pause nodepool so we can check the ordering of getting the nodes
        # and aquiring the semaphore.
        self.fake_nodepool.paused = True

        tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')

        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        self.assertEqual(
            len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")),
            0)

        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        self.assertEqual(
            len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")),
            1)

        self.fake_gerrit.addEvent(A.getChangeAbandonedEvent())
        self.waitUntilSettled()

        # The check pipeline should be empty
        items = self.getAllItems('tenant-one', 'check')
        self.assertEqual(len(items), 0)

        # The semaphore should be released
        self.assertEqual(
            len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")),
            0)

        self.executor_server.hold_jobs_in_build = False
        self.fake_nodepool.paused = False
        self.executor_server.release()
        self.waitUntilSettled()

    def test_semaphore_abandon_pending_execution(self):
        "Test abandon with job semaphores and pending job execution"

        # Pause the executor so it doesn't take any jobs.
        self.executor_server.pause()

        # Start merger as the paused executor won't take merge jobs.
        self._startMerger()

        # Pause nodepool so we can wait on the node requests and fulfill them
        # in a controlled manner.
        self.fake_nodepool.paused = True

        tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')

        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        self.assertEqual(
            len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")),
            0)

        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()
        reqs = list(self.scheds.first.sched.nodepool.getNodeRequests())
        self.assertEqual(len(reqs), 2)

        # Now unpause nodepool to fulfill the node requests. We cannot use
        # waitUntilSettled here because the executor is paused.
        self.fake_nodepool.paused = False
        for _ in iterate_timeout(30, 'fulfill node requests'):
            reqs = [
                r for r in self.scheds.first.sched.nodepool.getNodeRequests()
                if r.state != zuul.model.STATE_FULFILLED
            ]
            if len(reqs) == 0:
                break

        self.assertEqual(
            len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")),
            1)

        self.fake_gerrit.addEvent(A.getChangeAbandonedEvent())
        self.waitUntilSettled()

        # The check pipeline should be empty
        items = self.getAllItems('tenant-one', 'check')
        self.assertEqual(len(items), 0)

        # The semaphore should be released
        self.assertEqual(
            len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")),
            0)

        self.executor_server.release()
        self.waitUntilSettled()

    def test_semaphore_new_patchset(self):
        "Test new patchset with job semaphores"
        self.executor_server.hold_jobs_in_build = True
        tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')

        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        self.assertEqual(
            len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")),
            0)

        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        self.assertEqual(
            len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")),
            1)

        A.addPatchset()
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(2))
        self.waitUntilSettled()

        self.assertEqual(
            len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")),
            1)

        items = self.getAllItems('tenant-one', 'check')
        self.assertEqual(items[0].changes[0].number, '1')
        self.assertEqual(items[0].changes[0].patchset, '2')
        self.assertTrue(items[0].live)

        self.executor_server.hold_jobs_in_build = False
        self.executor_server.release()
        self.waitUntilSettled()

        # The semaphore should be released
        self.assertEqual(
            len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")),
            0)

    def test_semaphore_reconfigure(self):
        "Test reconfigure with job semaphores"
        self.executor_server.hold_jobs_in_build = True
        tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        self.assertEqual(
            len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")),
            0)

        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        self.assertEqual(
            len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")),
            1)

        # reconfigure without layout change
        self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
        self.waitUntilSettled()
        tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')

        # semaphore still must be held
        self.assertEqual(
            len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")),
            1)

        # remove the pipeline
        self.commitConfigUpdate(
            'common-config',
            'config/semaphore/zuul-reconfiguration.yaml')
        self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
        self.waitUntilSettled()
        tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')

        self.executor_server.release('project-test1')
        self.waitUntilSettled()

        # There should be no builds anymore
        self.assertEqual(len(self.builds), 0)

        # The semaphore should be released
        self.assertEqual(
            len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")),
            0)

    def test_semaphore_handler_cleanup(self):
        "Test the semaphore handler leak cleanup"
        self.executor_server.hold_jobs_in_build = True
        tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')

        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        self.assertEqual(
            len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")),
            0)

        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        self.assertEqual(
            len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")),
            1)

        # Save some variables for later use while the job is running
        item = self.getAllItems('tenant-one', 'check')[0]
        job = list(filter(lambda j: j.name == 'semaphore-one-test1',
                          item.getJobs()))[0]

        tenant.semaphore_handler.cleanupLeaks()

        # Nothing has leaked; our handle should be present.
        self.assertEqual(
            len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")),
            1)

        self.executor_server.hold_jobs_in_build = False
        self.executor_server.release()
        self.waitUntilSettled()

        # Make sure the semaphore is released normally
        self.assertEqual(
            len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")),
            0)

        # Use our previously saved data to simulate a leaked semaphore
        tenant.semaphore_handler.acquire(item, job, False)
        self.assertEqual(
            len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")),
            1)

        tenant.semaphore_handler.cleanupLeaks()
        # Make sure the leaked semaphore is cleaned up
        self.assertEqual(
            len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")),
            0)

    @simple_layout('layouts/multiple-semaphores.yaml')
    def test_multiple_semaphores(self):
        # Test a job with multiple semaphores
        self.executor_server.hold_jobs_in_build = True
        A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        # One job should be running, and hold sem1
        self.assertBuilds([dict(name='job1')])

        B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        # Job2 requires sem1 and sem2; it hasn't started because job
        # is still holding sem1.
        self.assertBuilds([dict(name='job1')])

        self.executor_server.release('job1')
        self.waitUntilSettled()

        # Job1 is finished, so job2 can acquire both semaphores.
        self.assertBuilds([dict(name='job2')])

        self.executor_server.release()
        self.waitUntilSettled()
        self.assertHistory([
            dict(name='job1', result='SUCCESS', changes='1,1'),
            dict(name='job2', result='SUCCESS', changes='2,1'),
        ])
        # TODO(corvus): Consider a version of this test which launches
        # 2 jobs with the same multiple-semaphore requirements
        # simultaneously to test the behavior with contention (at
        # least one should be able to start on each pass through the
        # loop).

    @simple_layout('layouts/semaphore-multi-pipeline.yaml')
    def test_semaphore_multi_pipeline(self):
        "Test semaphores in multiple pipelines"
        tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')

        self.executor_server.hold_jobs_in_build = True

        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        self.assertEqual(
            len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")),
            0)

        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        self.assertEqual(len(self.builds), 1)
        self.assertEqual(
            len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")),
            1)

        # Start a second change in a different pipeline
        B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
        B.addApproval('Code-Review', 2)
        self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
        self.waitUntilSettled()

        # Still just the first change holds the lock
        self.assertEqual(len(self.builds), 1)
        self.assertEqual(
            len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")),
            1)

        # Now the second should run
        self.executor_server.hold_jobs_in_build = False
        self.executor_server.release()
        self.waitUntilSettled()
        self.assertHistory([
            dict(name='check-job', result='SUCCESS', changes='1,1'),
            dict(name='gate-job', result='SUCCESS', changes='2,1'),
        ])


class TestSemaphoreMultiTenant(ZuulTestCase):
    tenant_config_file = 'config/multi-tenant-semaphore/main.yaml'

    def test_semaphore_tenant_isolation(self):
        "Test semaphores in multiple tenants"

        self.waitUntilSettled()
        tenant_one = self.scheds.first.sched.abide.tenants.get('tenant-one')
        tenant_two = self.scheds.first.sched.abide.tenants.get('tenant-two')

        self.executor_server.hold_jobs_in_build = True
        A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
        B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
        C = self.fake_gerrit.addFakeChange('org/project2', 'master', 'C')
        D = self.fake_gerrit.addFakeChange('org/project2', 'master', 'D')
        E = self.fake_gerrit.addFakeChange('org/project2', 'master', 'E')
        self.assertEqual(
            len(tenant_one.semaphore_handler.semaphoreHolders(
                "test-semaphore")), 0)
        self.assertEqual(
            len(tenant_two.semaphore_handler.semaphoreHolders(
                "test-semaphore")), 0)

        # add patches to project1 of tenant-one
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        # one build of project1-test1 must run
        # semaphore of tenant-one must be acquired once
        # semaphore of tenant-two must not be acquired
        self.assertEqual(len(self.builds), 1)
        self.assertEqual(self.builds[0].name, 'project1-test1')
        self.assertEqual(
            len(tenant_one.semaphore_handler.semaphoreHolders(
                "test-semaphore")), 1)
        self.assertEqual(
            len(tenant_two.semaphore_handler.semaphoreHolders(
                "test-semaphore")), 0)

        # add patches to project2 of tenant-two
        self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
        self.fake_gerrit.addEvent(D.getPatchsetCreatedEvent(1))
        self.fake_gerrit.addEvent(E.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        # one build of project1-test1 must run
        # two builds of project2-test1 must run
        # semaphore of tenant-one must be acquired once
        # semaphore of tenant-two must be acquired twice
        self.assertEqual(len(self.builds), 3)
        self.assertEqual(self.builds[0].name, 'project1-test1')
        self.assertEqual(self.builds[1].name, 'project2-test1')
        self.assertEqual(self.builds[2].name, 'project2-test1')
        self.assertEqual(
            len(tenant_one.semaphore_handler.semaphoreHolders(
                "test-semaphore")), 1)
        self.assertEqual(
            len(tenant_two.semaphore_handler.semaphoreHolders(
                "test-semaphore")), 2)

        self.executor_server.release('project1-test1')
        self.waitUntilSettled()

        # one build of project1-test1 must run
        # two builds of project2-test1 must run
        # semaphore of tenant-one must be acquired once
        # semaphore of tenant-two must be acquired twice
        self.assertEqual(len(self.builds), 3)
        self.assertEqual(self.builds[0].name, 'project2-test1')
        self.assertEqual(self.builds[1].name, 'project2-test1')
        self.assertEqual(self.builds[2].name, 'project1-test1')
        self.assertEqual(
            len(tenant_one.semaphore_handler.semaphoreHolders(
                "test-semaphore")), 1)
        self.assertEqual(
            len(tenant_two.semaphore_handler.semaphoreHolders(
                "test-semaphore")), 2)

        self.executor_server.release('project2-test1')
        self.waitUntilSettled()

        # one build of project1-test1 must run
        # one build of project2-test1 must run
        # semaphore of tenant-one must be acquired once
        # semaphore of tenant-two must be acquired once
        self.assertEqual(len(self.builds), 2)
        self.assertEqual(
            len(tenant_one.semaphore_handler.semaphoreHolders(
                "test-semaphore")), 1)
        self.assertEqual(
            len(tenant_two.semaphore_handler.semaphoreHolders(
                "test-semaphore")), 1)

        self.executor_server.hold_jobs_in_build = False
        self.executor_server.release()

        self.waitUntilSettled()

        # no build must run
        # semaphore of tenant-one must not be acquired
        # semaphore of tenant-two must not be acquired
        self.assertEqual(len(self.builds), 0)
        self.assertEqual(
            len(tenant_one.semaphore_handler.semaphoreHolders(
                "test-semaphore")), 0)
        self.assertEqual(
            len(tenant_two.semaphore_handler.semaphoreHolders(
                "test-semaphore")), 0)

        self.assertEqual(A.reported, 1)
        self.assertEqual(B.reported, 1)


class TestImplicitProject(ZuulTestCase):
    tenant_config_file = 'config/implicit-project/main.yaml'

    def test_implicit_project(self):
        # config project should work with implicit project name
        A = self.fake_gerrit.addFakeChange('common-config', 'master', 'A')
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

        # untrusted project should work with implicit project name
        B = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))

        self.waitUntilSettled()

        self.assertEqual(A.data['status'], 'NEW')
        self.assertEqual(A.reported, 1)
        self.assertEqual(B.data['status'], 'NEW')
        self.assertEqual(B.reported, 1)
        self.assertHistory([
            dict(name='test-common', result='SUCCESS', changes='1,1'),
            dict(name='test-common', result='SUCCESS', changes='2,1'),
            dict(name='test-project', result='SUCCESS', changes='2,1'),
        ], ordered=False)

        # now test adding a further project in repo
        in_repo_conf = textwrap.dedent(
            """
            - job:
                name: test-project
                run: playbooks/test-project.yaml
            - job:
                name: test2-project
                run: playbooks/test-project.yaml

            - project:
                check:
                  jobs:
                    - test-project
                gate:
                  jobs:
                    - test-project

            - project:
                check:
                  jobs:
                    - test2-project
                gate:
                  jobs:
                    - test2-project

            """)
        file_dict = {'.zuul.yaml': in_repo_conf}
        C = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
                                           files=file_dict)
        C.addApproval('Code-Review', 2)
        self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
        self.waitUntilSettled()

        # change C must be merged
        self.assertEqual(C.data['status'], 'MERGED')
        self.assertEqual(C.reported, 2)
        self.assertHistory([
            dict(name='test-common', result='SUCCESS', changes='1,1'),
            dict(name='test-common', result='SUCCESS', changes='2,1'),
            dict(name='test-project', result='SUCCESS', changes='2,1'),
            dict(name='test-common', result='SUCCESS', changes='3,1'),
            dict(name='test-project', result='SUCCESS', changes='3,1'),
            dict(name='test2-project', result='SUCCESS', changes='3,1'),
        ], ordered=False)


class TestSemaphoreInRepo(ZuulTestCase):
    config_file = 'zuul-connections-gerrit-and-github.conf'
    tenant_config_file = 'config/in-repo/main.yaml'

    def test_semaphore_in_repo(self):
        "Test semaphores in repo config"

        # This tests dynamic semaphore handling in project repos. The semaphore
        # max value should not be evaluated dynamically but must be updated
        # after the change lands.

        self.waitUntilSettled()
        tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')

        in_repo_conf = textwrap.dedent(
            """
            - job:
                name: project-test1

            - job:
                name: project-test2
                run: playbooks/project-test2.yaml
                semaphore: test-semaphore

            - project:
                name: org/project
                tenant-one-gate:
                  jobs:
                    - project-test2

            # the max value in dynamic layout must be ignored
            - semaphore:
                name: test-semaphore
                max: 2
            """)

        in_repo_playbook = textwrap.dedent(
            """
            - hosts: all
              tasks: []
            """)

        file_dict = {'.zuul.yaml': in_repo_conf,
                     'playbooks/project-test2.yaml': in_repo_playbook}
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
                                           files=file_dict)
        B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
        C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
        B.setDependsOn(A, 1)
        C.setDependsOn(A, 1)

        self.executor_server.hold_jobs_in_build = True

        A.addApproval('Code-Review', 2)
        B.addApproval('Code-Review', 2)
        C.addApproval('Code-Review', 2)
        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
        self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
        self.waitUntilSettled()

        # check that the layout in a queue item still has max value of 1
        # for test-semaphore
        manager = tenant.layout.pipeline_managers.get('tenant-one-gate')
        queue = None
        for queue_candidate in manager.state.queues:
            if queue_candidate.name == 'org/project':
                queue = queue_candidate
                break
        queue_item = queue.queue[0]
        item_dynamic_layout = manager._layout_cache.get(
            queue_item.layout_uuid)
        self.assertIsNotNone(item_dynamic_layout)
        dynamic_test_semaphore = item_dynamic_layout.getSemaphore(
            self.scheds.first.sched.abide, 'test-semaphore')
        self.assertEqual(dynamic_test_semaphore.max, 1)

        # one build must be in queue, one semaphores acquired
        self.assertEqual(len(self.builds), 1)
        self.assertEqual(self.builds[0].name, 'project-test2')
        self.assertEqual(
            len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")),
            1)

        self.executor_server.release('project-test2')
        self.waitUntilSettled()

        # change A must be merged
        self.assertEqual(A.data['status'], 'MERGED')
        self.assertEqual(A.reported, 2)

        # send change-merged event as the gerrit mock doesn't send it
        self.fake_gerrit.addEvent(A.getChangeMergedEvent())
        self.waitUntilSettled()

        # now that change A was merged, the new semaphore max must be effective
        tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
        self.assertEqual(tenant.layout.getSemaphore(
            self.scheds.first.sched.abide, 'test-semaphore').max, 2)

        # two builds must be in queue, two semaphores acquired
        self.assertEqual(len(self.builds), 2)
        self.assertEqual(self.builds[0].name, 'project-test2')
        self.assertEqual(self.builds[1].name, 'project-test2')
        self.assertEqual(
            len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")),
            2)

        self.executor_server.release('project-test2')
        self.waitUntilSettled()

        self.assertEqual(len(self.builds), 0)
        self.assertEqual(
            len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")),
            0
        )

        self.executor_server.hold_jobs_in_build = False
        self.executor_server.release()

        self.waitUntilSettled()
        self.assertEqual(len(self.builds), 0)

        self.assertEqual(A.reported, 2)
        self.assertEqual(B.reported, 2)
        self.assertEqual(C.reported, 2)


class TestSchedulerBranchMatcher(ZuulTestCase):

    @simple_layout('layouts/matcher-test.yaml')
    def test_job_branch_ignored(self):
        '''
        Test that branch matching logic works.

        The 'ignore-branch' job has a branch matcher that is supposed to
        match every branch except for the 'featureA' branch, so it should
        not be run on a change to that branch.
        '''
        self.create_branch('org/project', 'featureA')
        self.fake_gerrit.addEvent(
            self.fake_gerrit.getFakeBranchCreatedEvent(
                'org/project', 'featureA'))
        self.waitUntilSettled()
        A = self.fake_gerrit.addFakeChange('org/project', 'featureA', 'A')
        A.addApproval('Code-Review', 2)
        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.waitUntilSettled()
        self.printHistory()
        self.assertEqual(self.getJobFromHistory('project-test1').result,
                         'SUCCESS')
        self.assertJobNotInHistory('ignore-branch')
        self.assertEqual(A.data['status'], 'MERGED')
        self.assertEqual(A.reported, 2,
                         "A should report start and success")
        self.assertIn('gate', A.messages[1],
                      "A should transit gate")


class TestSchedulerFailFast(ZuulTestCase):
    tenant_config_file = 'config/fail-fast/main.yaml'

    def test_fail_fast(self):
        """
        Tests that a pipeline that is flagged with fail-fast
        aborts jobs early.
        """
        self.executor_server.hold_jobs_in_build = True
        self.fake_nodepool.pause()

        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        self.executor_server.failJob('project-test1', A)
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        self.waitUntilSettled()
        self.assertEqual(len(self.builds), 1)
        self.assertEqual(self.builds[0].name, 'project-merge')
        self.executor_server.release('project-merge')
        self.waitUntilSettled()

        # Now project-test1 and project-test2 should be running
        self.assertEqual(len(self.builds), 2)

        # Release project-test1 which will fail
        self.executor_server.release('project-test1')
        self.waitUntilSettled()

        self.fake_nodepool.unpause()
        self.waitUntilSettled()

        # Now project-test2 must be aborted
        self.assertEqual(len(self.builds), 0)
        self.assertEqual(A.reported, 1)
        self.assertHistory([
            dict(name='project-merge', result='SUCCESS', changes='1,1'),
            dict(name='project-test1', result='FAILURE', changes='1,1'),
            dict(name='project-test2', result='ABORTED', changes='1,1'),
        ], ordered=False)

    def test_fail_fast_gate(self):
        """
        Tests that a pipeline that is flagged with fail-fast
        aborts jobs early.
        """
        self.executor_server.hold_jobs_in_build = True
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        A.addApproval('Code-Review', 2)
        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.waitUntilSettled()

        B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
        self.executor_server.failJob('project-test1', B)
        B.addApproval('Code-Review', 2)
        self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
        self.waitUntilSettled()

        self.assertEqual(len(self.builds), 4)
        # Release project-test1 which will fail
        self.builds[2].release()
        self.waitUntilSettled()

        # We should only have the builds from change A now
        self.assertEqual(len(self.builds), 2)
        self.assertEqual(self.builds[0].name, 'project-test1')
        self.assertEqual(self.builds[1].name, 'project-test2')

        # But both changes should still be in the pipeline
        items = self.getAllItems('tenant-one', 'gate')
        self.assertEqual(len(items), 2)
        self.assertEqual(A.reported, 1)
        self.assertEqual(B.reported, 1)

        # Release change A
        self.executor_server.hold_jobs_in_build = False
        self.executor_server.release()
        self.waitUntilSettled()

        self.assertEqual(len(self.builds), 0)
        self.assertEqual(A.reported, 2)
        self.assertEqual(B.reported, 2)
        self.assertHistory([
            dict(name='project-test1', result='SUCCESS', changes='1,1'),
            dict(name='project-test2', result='SUCCESS', changes='1,1'),
            dict(name='project-test1', result='FAILURE', changes='1,1 2,1'),
            dict(name='project-test2', result='ABORTED', changes='1,1 2,1'),
        ], ordered=False)

    def test_fail_fast_nonvoting(self):
        """
        Tests that a pipeline that is flagged with fail-fast
        doesn't abort jobs due to a non-voting job.
        """
        self.executor_server.hold_jobs_in_build = True

        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        self.executor_server.failJob('project-test6', A)
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        self.waitUntilSettled()
        self.assertEqual(len(self.builds), 2)
        self.assertEqual(self.builds[0].name, 'project-merge')
        self.executor_server.release('project-merge')
        self.waitUntilSettled()

        # Now project-test1, project-test2, project-test5 and project-test6
        # should be running
        self.assertEqual(len(self.builds), 4)

        # Release project-test6 which will fail
        self.executor_server.release('project-test6')
        self.waitUntilSettled()

        # Now project-test1, project-test2 and project-test5 should be running
        self.assertEqual(len(self.builds), 3)

        self.executor_server.hold_jobs_in_build = False
        self.executor_server.release()
        self.waitUntilSettled()

        self.assertEqual(len(self.builds), 0)
        self.assertEqual(A.reported, 1)
        self.assertHistory([
            dict(name='project-merge', result='SUCCESS', changes='1,1'),
            dict(name='project-test1', result='SUCCESS', changes='1,1'),
            dict(name='project-test2', result='SUCCESS', changes='1,1'),
            dict(name='project-test3', result='SUCCESS', changes='1,1'),
            dict(name='project-test4', result='SUCCESS', changes='1,1'),
            dict(name='project-test5', result='SUCCESS', changes='1,1'),
            dict(name='project-test6', result='FAILURE', changes='1,1'),
        ], ordered=False)

    def test_fail_fast_retry(self):
        """
        Tests that a retried build doesn't trigger fail-fast.
        """
        self.executor_server.hold_jobs_in_build = True

        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        # project-merge and project-test5
        self.assertEqual(len(self.builds), 2)

        # Force a retry of first build
        self.builds[0].requeue = True
        self.executor_server.hold_jobs_in_build = False
        self.executor_server.release()
        self.waitUntilSettled()

        self.assertEqual(len(self.builds), 0)
        self.assertEqual(A.reported, 1)
        self.assertHistory([
            dict(name='project-merge', result=None, changes='1,1'),
            dict(name='project-merge', result='SUCCESS', changes='1,1'),
            dict(name='project-test1', result='SUCCESS', changes='1,1'),
            dict(name='project-test2', result='SUCCESS', changes='1,1'),
            dict(name='project-test3', result='SUCCESS', changes='1,1'),
            dict(name='project-test4', result='SUCCESS', changes='1,1'),
            dict(name='project-test5', result='SUCCESS', changes='1,1'),
            dict(name='project-test6', result='SUCCESS', changes='1,1'),
        ], ordered=False)

    def test_fail_fast_node_failure(self):
        """
        Tests that a pipeline that is flagged with fail-fast
        aborts jobs early if a node request failed.
        """
        self.executor_server.hold_jobs_in_build = True
        self.fake_nodepool.pause()

        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        self.waitUntilSettled()
        self.assertEqual(len(self.builds), 1)
        self.assertEqual(self.builds[0].name, 'project-merge')
        self.executor_server.release('project-merge')
        self.waitUntilSettled()

        # Fail node request for project-test5
        request = self.fake_nodepool.getNodeRequests()[0]
        self.fake_nodepool.addFailRequest(request)
        self.fake_nodepool.unpause()
        self.waitUntilSettled()

        self.assertEqual(A.reported, 1)
        self.assertHistory([
            dict(name='project-merge', result='SUCCESS', changes='1,1'),
            dict(name='project-test1', result='ABORTED', changes='1,1'),
            dict(name='project-test2', result='ABORTED', changes='1,1'),
        ], ordered=False)

    def test_fail_fast_node_failure_nonvoting(self):
        """
        Tests that a pipeline that is flagged with fail-fast
        doesn't abort jobs due to a node failure for non-voting job.
        """
        self.executor_server.hold_jobs_in_build = True

        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        self.fake_nodepool.pause()
        self.assertEqual(len(self.builds), 2)
        self.assertEqual(self.builds[0].name, 'project-merge')
        self.executor_server.release('project-merge')
        self.executor_server.release('project-test5')
        self.waitUntilSettled()

        # Now project-test1 and project-test2 should be running
        self.assertEqual(len(self.builds), 2)

        # Fail node request for project-test6
        request = self.fake_nodepool.getNodeRequests()[0]
        self.fake_nodepool.addFailRequest(request)
        self.fake_nodepool.unpause()
        self.waitUntilSettled()

        self.executor_server.hold_jobs_in_build = False
        self.executor_server.release()
        self.waitUntilSettled()

        self.assertEqual(len(self.builds), 0)
        self.assertEqual(A.reported, 1)
        self.assertHistory([
            dict(name='project-merge', result='SUCCESS', changes='1,1'),
            dict(name='project-test1', result='SUCCESS', changes='1,1'),
            dict(name='project-test2', result='SUCCESS', changes='1,1'),
            dict(name='project-test3', result='SUCCESS', changes='1,1'),
            dict(name='project-test4', result='SUCCESS', changes='1,1'),
            dict(name='project-test5', result='SUCCESS', changes='1,1'),
        ], ordered=False)

    def test_fail_fast_no_cancel_completed(self):
        """
        Regression test to check that we are not cancelling jobs that
        have already completed, as this would overwrite valid return data
        (e.g. the log URL).
        """
        self.fake_nodepool.pause()
        self.executor_server.hold_jobs_in_build = True

        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        self.executor_server.failJob('project-test1', A)
        self.executor_server.returnData(
            "project-test2", A, {
                "zuul": {
                    "log_url": "some/log/url/",
                },
            }
        )
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        self.executor_server.release('project-merge')
        self.waitUntilSettled()
        self.assertEqual(len(self.builds), 2)

        # Release the failing build first so it is the first
        # result event to be processed.
        job_workers = self.executor_server.job_workers.copy()
        released = self.executor_server.release('project-test1')
        for _ in iterate_timeout(10, 'project-test1 to be released'):
            if len(self.builds) == 1:
                break
        fake_build = released[0]
        job = job_workers.get(fake_build.build_request.uuid)
        job.wait()
        # Release successful build and wait for it to be gone,
        # so both result events are processed in the same iteration.
        released = self.executor_server.release('project-test2')
        for _ in iterate_timeout(10, 'project-test2 to be released'):
            if len(self.builds) == 0:
                break
        fake_build = released[0]
        job = job_workers.get(fake_build.build_request.uuid)
        job.wait()

        self.executor_server.hold_jobs_in_build = False
        self.fake_nodepool.unpause()
        self.waitUntilSettled()

        self.assertEqual(len(self.builds), 0)
        self.assertEqual(A.reported, 1)
        self.assertHistory([
            dict(name='project-merge', result='SUCCESS', changes='1,1'),
            dict(name='project-test1', result='FAILURE', changes='1,1'),
            dict(name='project-test2', result='SUCCESS', changes='1,1'),
        ], ordered=False)

        db = self.scheds.first.connections.connections['database']

        builds = db.getBuilds()
        self.assertEqual(len(builds), 3)

        # Make sure the build is reported as cancelled and that
        # the log URL in the DB is correct.
        build = builds[0]
        self.assertEqual(build.result, "CANCELED")
        self.assertEqual(build.log_url, "some/log/url/")


class TestPipelineSupersedes(ZuulTestCase):

    @simple_layout('layouts/pipeline-supercedes.yaml')
    def test_supercedes(self):
        """
        Tests that a pipeline that is flagged with fail-fast
        aborts jobs early.
        """
        self.executor_server.hold_jobs_in_build = True

        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        self.assertEqual(len(self.builds), 1)
        self.assertEqual(self.builds[0].name, 'test-job')

        A.addApproval('Code-Review', 2)
        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.waitUntilSettled()

        self.assertEqual(len(self.builds), 1)
        self.assertEqual(self.builds[0].name, 'test-job')
        self.assertEqual(self.builds[0].pipeline, 'gate')

        self.executor_server.hold_jobs_in_build = False
        self.executor_server.release()
        self.waitUntilSettled()

        self.assertEqual(len(self.builds), 0)
        self.assertEqual(A.reported, 2)
        self.assertEqual(A.data['status'], 'MERGED')
        self.assertHistory([
            dict(name='test-job', result='ABORTED', changes='1,1'),
            dict(name='test-job', result='SUCCESS', changes='1,1'),
        ], ordered=False)


class TestSchedulerExcludeAll(ZuulTestCase):
    tenant_config_file = 'config/two-tenant/exclude-all.yaml'

    def test_skip_reconfig_exclude_all(self):
        """Test that we don't trigger a reconfiguration for a tenant
        when the changed project excludes all config."""
        config = textwrap.dedent(
            """
            - job:
                name: project2-test
                parent: test

            - project:
                check:
                  jobs:
                    - project2-test
            """)
        file_dict = {'zuul.yaml': config}
        A = self.fake_gerrit.addFakeChange('org/project2', 'master', 'A',
                                           files=file_dict)
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()
        self.assertHistory([
            dict(name='project2-test', result='SUCCESS', changes='1,1'),
        ])

        sched = self.scheds.first.sched
        tenant_one_layout_state = sched.local_layout_state["tenant-one"]
        tenant_two_layout_state = sched.local_layout_state["tenant-two"]

        A.setMerged()
        self.fake_gerrit.addEvent(A.getChangeMergedEvent())
        self.waitUntilSettled()

        # We don't expect a reconfiguration for tenant-one as it excludes
        # all config of org/project2.
        self.assertEqual(sched.local_layout_state["tenant-one"],
                         tenant_one_layout_state)
        # As tenant-two includes the config from org/project2, the merge of
        # change A should have triggered a reconfig.
        self.assertGreater(sched.local_layout_state["tenant-two"],
                           tenant_two_layout_state)

        B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()
        self.assertHistory([
            dict(name='project2-test', result='SUCCESS', changes='1,1'),
            dict(name='project2-test', result='SUCCESS', changes='2,1'),
        ])


class TestReportBuildPage(ZuulTestCase):
    tenant_config_file = 'config/build-page/main.yaml'

    def test_tenant_url(self):
        """
        Test that the tenant url is used in reporting the build page.
        """
        A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()
        self.assertHistory([
            dict(name='python27', result='SUCCESS', changes='1,1'),
        ])
        self.assertIn('python27 https://one.example.com/build/',
                      A.messages[0])

    def test_base_url(self):
        """
        Test that the web base url is used in reporting the build page.
        """
        A = self.fake_gerrit.addFakeChange('org/project2', 'master', 'A')
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()
        self.assertHistory([
            dict(name='python27', result='SUCCESS', changes='1,1'),
        ])
        self.assertIn('python27 https://zuul.example.com/t/tenant-two/build/',
                      A.messages[0])

    def test_no_build_page(self):
        """
        Test that we fall back to the old behavior if the tenant is
        not configured to report the build page
        """
        A = self.fake_gerrit.addFakeChange('org/project3', 'master', 'A')
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()
        self.assertHistory([
            dict(name='python27', result='SUCCESS', changes='1,1'),
        ])
        self.assertIn('python27 https://', A.messages[0])


class TestSchedulerSmartReconfiguration(ZuulTestCase):
    tenant_config_file = 'config/multi-tenant/main.yaml'

    def _test_smart_reconfiguration(self, command_socket=False):
        """
        Tests that smart reconfiguration works

        In this scenario we have the tenants tenant-one, tenant-two and
        tenant-three. We make the following changes and then trigger a smart
        reconfiguration:
        - tenant-one remains unchanged
        - tenant-two gets another repo
        - tenant-three gets removed completely
        - tenant-four is a new tenant
        """
        self.executor_server.hold_jobs_in_build = True

        # Create changes for all tenants
        A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
        C = self.fake_gerrit.addFakeChange('org/project3', 'master', 'C')
        self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))

        # record previous tenant reconfiguration time, which may not be set
        old_one = self.scheds.first.sched.tenant_layout_state.get(
            'tenant-one', EMPTY_LAYOUT_STATE)
        old_two = self.scheds.first.sched.tenant_layout_state.get(
            'tenant-two', EMPTY_LAYOUT_STATE)
        self.waitUntilSettled()

        self.newTenantConfig('config/multi-tenant/main-reconfig.yaml')

        del self.merge_job_history
        self.scheds.execute(
            lambda app: app.smartReconfigure(command_socket=command_socket))

        # Wait for smart reconfiguration. Only tenant-two should be
        # reconfigured. Note that waitUntilSettled is not
        # reliable here because the reconfigure event may arrive in the
        # event queue after waitUntilSettled.
        start = time.time()
        while True:
            if time.time() - start > 30:
                raise Exception("Timeout waiting for smart reconfiguration")
            new_two = self.scheds.first.sched.tenant_layout_state.get(
                'tenant-two', EMPTY_LAYOUT_STATE)
            if old_two < new_two:
                break
            else:
                time.sleep(0.1)

        self.waitUntilSettled()

        # We're only adding two new repos, so we should only need to
        # issue 2 cat jobs.
        cat_jobs = self.merge_job_history.get(zuul.model.MergeRequest.CAT)
        self.assertEqual(len(cat_jobs), 2)

        # Ensure that tenant-one has not been reconfigured
        new_one = self.scheds.first.sched.tenant_layout_state.get(
            'tenant-one', EMPTY_LAYOUT_STATE)
        self.assertEqual(old_one, new_one)

        self.executor_server.hold_jobs_in_build = False
        self.executor_server.release()
        self.waitUntilSettled()

        # Changes in tenant-one and tenant-two have to be reported
        self.assertEqual(1, A.reported)
        self.assertEqual(1, B.reported)

        # The tenant-three has been removed so nothing should be reported
        self.assertEqual(0, C.reported)
        self.assertNotIn('tenant-three', self.scheds.first.sched.abide.tenants)

        # Verify known tenants
        expected_tenants = {'tenant-one', 'tenant-two', 'tenant-four'}
        self.assertEqual(expected_tenants,
                         self.scheds.first.sched.abide.tenants.keys())

        self.assertIsNotNone(
            self.scheds.first.sched.tenant_layout_state.get('tenant-four'),
            'Tenant tenant-four should exist now.'
        )

        # Test that the new tenant-four actually works
        D = self.fake_gerrit.addFakeChange('org/project4', 'master', 'D')
        self.fake_gerrit.addEvent(D.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()
        self.assertEqual(1, D.reported)

        # Test that the new project in tenant-two works
        B2 = self.fake_gerrit.addFakeChange('org/project2b', 'master', 'B2')
        self.fake_gerrit.addEvent(B2.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()
        self.assertEqual(1, B2.reported)

        events = self.scheds.first.sched.sql.connection.getSystemEvents()
        self.assertEqual(3, len(events))
        for x in range(3):
            self.assertEqual("tenant reconfiguration", events[x].event_type)
            self.assertEqual("Tenant reconfiguration due to external request",
                             events[x].description)

    def test_smart_reconfiguration(self):
        "Test that live reconfiguration works"
        self._test_smart_reconfiguration()

    def test_smart_reconfiguration_command_socket(self):
        "Test that live reconfiguration works using command socket"
        self._test_smart_reconfiguration(command_socket=True)


class TestReconfigureBranch(ZuulTestCase):

    def _setupTenantReconfigureTime(self):
        self.old = self.scheds.first.sched.tenant_layout_state.get(
            'tenant-one', EMPTY_LAYOUT_STATE)

    def _createBranch(self):
        self.create_branch('org/project1', 'stable')
        self.fake_gerrit.addEvent(
            self.fake_gerrit.getFakeBranchCreatedEvent(
                'org/project1', 'stable'))
        self.waitUntilSettled()

    def _deleteBranch(self):
        self.delete_branch('org/project1', 'stable')
        self.fake_gerrit.addEvent(
            self.fake_gerrit.getFakeBranchDeletedEvent(
                'org/project1', 'stable'))
        self.waitUntilSettled()

    def _expectReconfigure(self, doReconfigure):
        new = self.scheds.first.sched.tenant_layout_state.get(
            'tenant-one', EMPTY_LAYOUT_STATE)
        if doReconfigure:
            self.assertLess(self.old, new)
        else:
            self.assertEqual(self.old, new)
        self.old = new


class TestReconfigureBranchCreateDeleteSshHttp(TestReconfigureBranch):
    tenant_config_file = 'config/single-tenant/main.yaml'
    config_file = 'zuul-gerrit-web.conf'

    def test_reconfigure_cache_branch_create_delete(self):
        "Test that cache is updated clear on branch creation/deletion"
        self._setupTenantReconfigureTime()
        self._createBranch()
        self._expectReconfigure(True)
        self._deleteBranch()
        self._expectReconfigure(True)


class TestReconfigureBranchCreateDeleteSsh(TestReconfigureBranch):
    tenant_config_file = 'config/single-tenant/main.yaml'

    def test_reconfigure_cache_branch_create_delete(self):
        "Test that cache is updated clear on branch creation/deletion"
        self._setupTenantReconfigureTime()
        self._createBranch()
        self._expectReconfigure(True)
        self._deleteBranch()
        self._expectReconfigure(True)


class TestReconfigureBranchCreateDeleteHttp(TestReconfigureBranch):
    tenant_config_file = 'config/single-tenant/main.yaml'
    config_file = 'zuul-gerrit-no-stream.conf'

    def test_reconfigure_cache_branch_create_delete(self):
        "Test that cache is updated clear on branch creation/deletion"
        self._setupTenantReconfigureTime()
        self._createBranch()
        self._expectReconfigure(True)
        self._deleteBranch()
        self._expectReconfigure(True)


class TestEventProcessing(ZuulTestCase):
    tenant_config_file = 'config/event-processing/main.yaml'

    # Some regression tests for ZK-distributed event processing

    def test_independent_tenants(self):
        # Test that an exception in one tenant doesn't break others

        orig = zuul.scheduler.Scheduler._forward_trigger_event

        def patched_forward(obj, *args, **kw):
            if args[1].name == 'tenant-one':
                raise Exception("test")
            return orig(obj, *args, **kw)

        self.useFixture(fixtures.MonkeyPatch(
            'zuul.scheduler.Scheduler._forward_trigger_event',
            patched_forward))

        A = self.fake_gerrit.addFakeChange('org/project2', 'master', 'A')
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()
        self.assertHistory([
            dict(name='checkjob', result='SUCCESS', changes='1,1'),
        ], ordered=False)

    @skip("Failure can only be detected in logs; see test_ref_equality")
    def test_change_types(self):
        # Test that when we decide whether to forward events, we can
        # compare items with different change types (branch vs
        # change).

        # We can't detect a failure here except by observing the logs;
        # this test is left in case that's useful in the future, but
        # automated detection of this failure case is handled by
        # test_ref_equality.

        # Enqueue a tag
        self.executor_server.hold_jobs_in_build = True
        event = self.fake_gerrit.addFakeTag('org/project1', 'master', 'foo')
        self.fake_gerrit.addEvent(event)

        # Enqueue a change and make sure the scheduler is able to
        # compare the two when forwarding the event
        A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
        self.waitUntilSettled()

        self.executor_server.hold_jobs_in_build = False
        self.executor_server.release()
        self.waitUntilSettled()

        self.assertHistory([
            dict(name='tagjob', result='SUCCESS'),
            dict(name='checkjob', result='SUCCESS', changes='1,1'),
        ], ordered=False)


class TestWaitForInit(ZuulTestCase):
    tenant_config_file = 'config/single-tenant/main.yaml'
    wait_for_init = True

    def setUp(self):
        with self.assertLogs('zuul.Scheduler-0', level='DEBUG') as full_logs:
            super().setUp()
            self.assertRegexInList('Waiting for tenant initialization',
                                   full_logs.output)

    def test_wait_for_init(self):
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        A.addApproval('Code-Review', 2)
        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.waitUntilSettled()
        self.assertHistory([
            dict(name='project-merge', result='SUCCESS', changes='1,1'),
            dict(name='project-test1', result='SUCCESS', changes='1,1'),
            dict(name='project-test2', result='SUCCESS', changes='1,1'),
        ], ordered=False)


class TestDisablePipelines(ZuulTestCase):
    tenant_config_file = 'config/single-tenant/main.yaml'
    disable_pipelines = True

    def test_disable_pipelines(self):
        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
        A.addApproval('Code-Review', 2)
        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
        self.waitUntilSettled()
        self.assertHistory([])
