# Copyright 2015 Cisco Systems, Inc.  All rights reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import copy
import random

from oslo_config import cfg
from oslo_log import log as logging
from oslo_utils import uuidutils
import sqlalchemy as sa
from sqlalchemy import orm
from sqlalchemy.orm import exc
from sqlalchemy.orm import joinedload
from sqlalchemy.sql import expression as expr

from neutron.db import models_v2
from neutron_lib import exceptions as n_exc

from networking_cisco._i18n import _
from networking_cisco import backwards_compatibility as bc
from networking_cisco.backwards_compatibility import extensions
from networking_cisco.backwards_compatibility import l3_const
from networking_cisco.backwards_compatibility import l3_exceptions
from networking_cisco.plugins.cisco.common import cisco_constants
from networking_cisco.plugins.cisco.common import utils as cisco_utils
from networking_cisco.plugins.cisco.extensions import ha
from networking_cisco.plugins.cisco.extensions import routerrole
from networking_cisco.plugins.cisco.extensions import routertype

LOG = logging.getLogger(__name__)


RouterPort = bc.RouterPort

HA_GROUP = 'group'
HA_PORT = 'ha_port'

MAX_VRRP_GROUPS = 4094
MAX_HSRP_GROUPS = 4094
MAX_GLBP_GROUPS = 1023

is_attr_set = bc.is_attr_set
ATTR_NOT_SPECIFIED = bc.constants.ATTR_NOT_SPECIFIED
EXTERNAL_GW_INFO = l3_const.EXTERNAL_GW_INFO
DEVICE_OWNER_ROUTER_GW = bc.constants.DEVICE_OWNER_ROUTER_GW
DEVICE_OWNER_ROUTER_INTF = bc.constants.DEVICE_OWNER_ROUTER_INTF
DEVICE_OWNER_ROUTER_HA_INTF = bc.constants.DEVICE_OWNER_ROUTER_HA_INTF
ROUTER_ROLE_HA_REDUNDANCY = cisco_constants.ROUTER_ROLE_HA_REDUNDANCY

DEFAULT_MASTER_PRIORITY = 100
PRIORITY_INCREASE_STEP = -3
REDUNDANCY_ROUTER_SUFFIX = '_HA_backup_'
DEFAULT_PING_INTERVAL = 5
PROBE_TARGET_OPT_NAME = 'default_probe_target'

LOOKUP_RETRIES = 5
RETRY_INTERVAL = 3

router_appliance_opts = [
    cfg.BoolOpt('ha_support_enabled', default=True,
                help=_("Enables high-availability support")),
    cfg.BoolOpt('ha_enabled_by_default', default=False,
                help=_("Enables high-availability functionality for Neutron "
                       "router even if user does not explicitly request it")),
    cfg.IntOpt('default_ha_redundancy_level', default=ha.MIN_REDUNDANCY_LEVEL,
               help=_("Default number of routers added for redundancy when "
                      "high-availability by VRRP, HSRP, or GLBP is used")),
    cfg.StrOpt('default_ha_mechanism', default=ha.HA_HSRP,
               help=_("Default mechanism used to implement "
                      "high-availability")),
    cfg.ListOpt('disabled_ha_mechanisms', default=[],
                help=_("List of administratively disabled high-availability "
                       "mechanisms (VRRP, HSRP, GBLP)")),
    cfg.BoolOpt('connectivity_probing_enabled_by_default', default=False,
                help=_("Enables connectivity probing for high-availability "
                       "even if user does not explicitly request it")),
    cfg.StrOpt('default_probe_target', default=None,
               help=_("Host that will be probe target for high-availability "
                      "connectivity probing if user does not specify it")),
    cfg.StrOpt('default_ping_interval', default=DEFAULT_PING_INTERVAL,
               help=_("Time (in seconds) between probes for high-availability "
                      "connectivity probing if user does not specify it")),
]

cfg.CONF.register_opts(router_appliance_opts, "ha")


class RouterHASetting(bc.model_base.BASEV2):
    """Represents HA settings for router visible to user."""
    __tablename__ = 'cisco_router_ha_settings'

    router_id = sa.Column(sa.String(36),
                          sa.ForeignKey('routers.id', ondelete='CASCADE'),
                          primary_key=True)
    router = orm.relationship(
        bc.Router,
        backref=orm.backref('ha_settings', cascade='all', uselist=False))
    # 'ha_type' can be 'VRRP', 'HSRP', or 'GLBP'
    ha_type = sa.Column(sa.String(255))
    # 'redundancy_level' is number of extra routers for redundancy
    redundancy_level = sa.Column(sa.Integer,
                                 server_default=str(ha.MIN_REDUNDANCY_LEVEL))
    # 'priority' is the priority used in VRRP, HSRP, and GLBP
    priority = sa.Column(sa.Integer)
    # 'probe_connectivity' is True if ICMP echo pinging is enabled
    probe_connectivity = sa.Column(sa.Boolean)
    # 'probe_target' is ip address of host that is probed
    probe_target = sa.Column(sa.String(64))
    # 'ping_interval' is the time between probes
    probe_interval = sa.Column(sa.Integer)
    # 'state' is the state of the user visible router: HA_ACTIVE or HA_STANDBY
    state = sa.Column(sa.Enum(ha.HA_ACTIVE, ha.HA_STANDBY, name='ha_states'),
                      default=ha.HA_ACTIVE, server_default=ha.HA_ACTIVE)


class RouterHAGroup(bc.model_base.BASEV2, bc.model_base.HasId,
                    bc.HasProject):
    """Represents an HA group as used in VRRP, HSRP, and GLBP."""
    __tablename__ = 'cisco_router_ha_groups'

    # 'ha_type' can be 'VRRP', 'HSRP', or 'GLBP'
    ha_type = sa.Column(sa.String(255))
    # 'group_identity'
    group_identity = sa.Column(sa.String(255))
    # 'ha_port_id' is id of port used for virtual IP address
    ha_port_id = sa.Column(sa.String(36),
                           sa.ForeignKey('ports.id', ondelete='CASCADE'),
                           primary_key=True)
    ha_port = orm.relationship(
        models_v2.Port,
        primaryjoin='Port.id==RouterHAGroup.ha_port_id')
    # 'extra_port_id' is id of port for user visible router's extra ip address
    extra_port_id = sa.Column(sa.String(36),
                              sa.ForeignKey('ports.id', ondelete='SET NULL'),
                              nullable=True)
    extra_port = orm.relationship(
        models_v2.Port,
        primaryjoin='Port.id==RouterHAGroup.extra_port_id')
    # 'subnet_id' is id of subnet that this HA group serves
    subnet_id = sa.Column(sa.String(36),
                          sa.ForeignKey('subnets.id', ondelete='CASCADE'),
                          primary_key=True)
    # 'user_router_id' is id of router visible to the user
    user_router_id = sa.Column(sa.String(36),
                               sa.ForeignKey('routers.id'))
    # 'timers_config' holds timer specific configurations
    timers_config = sa.Column(sa.String(255))
    # 'tracking_config' holds tracking object specific configurations
    tracking_config = sa.Column(sa.String(255))
    # 'other_config' holds other method specific configurations
    other_config = sa.Column(sa.String(255))


class RouterRedundancyBinding(bc.model_base.BASEV2):
    """Represents binding between an HA enabled router and its
    redundancy routers.
    """
    __tablename__ = 'cisco_router_redundancy_bindings'

    # 'redundancy_router_id' is id of a redundancy router
    redundancy_router_id = sa.Column(sa.String(36),
                                     sa.ForeignKey('routers.id',
                                                   ondelete='CASCADE'),
                                     primary_key=True)
    redundancy_router = orm.relationship(
        bc.Router,
        primaryjoin='Router.id==RouterRedundancyBinding.redundancy_router_id',
        backref=orm.backref('redundancy_binding', cascade='save-update, merge',
                            passive_deletes='all', uselist=False))
    # 'priority' is the priority used in VRRP, HSRP, and GLBP
    priority = sa.Column(sa.Integer)
    # 'state' is the state of the redundancy router: HA_ACTIVE or HA_STANDBY
    state = sa.Column(sa.Enum(ha.HA_ACTIVE, ha.HA_STANDBY, name='ha_states'),
                      default=ha.HA_STANDBY, server_default=ha.HA_STANDBY)
    # 'user_router_id' is id of router visible to the user
    user_router_id = sa.Column(sa.String(36),
                               sa.ForeignKey('routers.id'))
    user_router = orm.relationship(
        bc.Router,
        primaryjoin='Router.id==RouterRedundancyBinding.user_router_id',
        backref=orm.backref('redundancy_bindings',
                            order_by=priority, cascade='all'))
    __mapper_args__ = {
        'confirm_deleted_rows': False
    }


class HA_db_mixin(object):
    """Mixin class to support VRRP, HSRP, and GLBP based HA for routing."""

    def _ensure_create_ha_compliant(self, router, router_type):
        """To be called in create_router() BEFORE router is created in DB."""
        details = router.pop(ha.DETAILS, {})
        if details == ATTR_NOT_SPECIFIED:
            details = {}
        res = {ha.ENABLED: router.pop(ha.ENABLED, ATTR_NOT_SPECIFIED),
               ha.DETAILS: details}

        if not is_attr_set(res[ha.ENABLED]):
            res[ha.ENABLED] = router_type['ha_enabled_by_default']
        if res[ha.ENABLED] and not cfg.CONF.ha.ha_support_enabled:
            raise ha.HADisabled()
        if not res[ha.ENABLED]:
            return res
        if not is_attr_set(details.get(ha.TYPE, ATTR_NOT_SPECIFIED)):
            details[ha.TYPE] = cfg.CONF.ha.default_ha_mechanism
        if details[ha.TYPE] in cfg.CONF.ha.disabled_ha_mechanisms:
            raise ha.HADisabledHAType(ha_type=details[ha.TYPE])
        if not is_attr_set(details.get(ha.REDUNDANCY_LEVEL,
                                       ATTR_NOT_SPECIFIED)):
            details[ha.REDUNDANCY_LEVEL] = (
                cfg.CONF.ha.default_ha_redundancy_level)
        if not is_attr_set(details.get(ha.PROBE_CONNECTIVITY,
                                       ATTR_NOT_SPECIFIED)):
            details[ha.PROBE_CONNECTIVITY] = (
                cfg.CONF.ha.connectivity_probing_enabled_by_default)
        if not is_attr_set(details.get(ha.PROBE_TARGET, ATTR_NOT_SPECIFIED)):
            details[ha.PROBE_TARGET] = cfg.CONF.ha.default_probe_target
        if not is_attr_set(details.get(ha.PROBE_INTERVAL, ATTR_NOT_SPECIFIED)):
            details[ha.PROBE_INTERVAL] = cfg.CONF.ha.default_ping_interval
        return res

    def _create_redundancy_routers(self, context, new_router, ha_settings,
                                   new_router_db, ports=None, expire_db=False):
        """To be called in create_router() AFTER router has been
        created in DB.
        """
        if (ha.ENABLED not in ha_settings or
                not ha_settings[ha.ENABLED]):
            new_router[ha.HA] = {ha.ENABLED: False}
            return
        ha_spec = ha_settings[ha.DETAILS]
        priority = ha_spec.get(ha.PRIORITY, DEFAULT_MASTER_PRIORITY)
        with context.session.begin(subtransactions=True):
            r_ha_s_db = RouterHASetting(
                router_id=new_router['id'],
                ha_type=ha_spec[ha.TYPE],
                redundancy_level=ha_spec[ha.REDUNDANCY_LEVEL],
                priority=priority,
                probe_connectivity=ha_spec[ha.PROBE_CONNECTIVITY],
                probe_target=ha_spec[ha.PROBE_TARGET],
                probe_interval=ha_spec[ha.PROBE_INTERVAL])
            context.session.add(r_ha_s_db)
        if r_ha_s_db.probe_connectivity and r_ha_s_db.probe_target is None:
            LOG.warning("Connectivity probing for high-availability is "
                        "enabled but probe target is not specified. Please"
                        " configure option \'default_probe_target\'.")
        e_context = context.elevated()
        if new_router_db.gw_port:
            # generate ha settings and extra port for router gateway (VIP) port
            gw_port = self._core_plugin._make_port_dict(new_router_db.gw_port)
            self._create_ha_group(e_context, new_router, gw_port, r_ha_s_db)
        self._add_redundancy_routers(e_context, 1,
                                     ha_spec[ha.REDUNDANCY_LEVEL] + 1,
                                     new_router, ports or [], r_ha_s_db)
        if expire_db:
            context.session.expire(new_router_db)
        self._extend_router_dict_ha(new_router, new_router_db)

    def _ensure_update_ha_compliant(self, router, current_router,
                                    r_hd_binding_db):
        """To be called in update_router() BEFORE router has been
        updated in DB.
        """
        if r_hd_binding_db.role == ROUTER_ROLE_HA_REDUNDANCY:
            return {ha.ENABLED: False}
        auto_enable_ha = r_hd_binding_db.router_type.ha_enabled_by_default
        requested_ha_details = router.pop(ha.DETAILS, {})
        # If ha_details are given then ha is assumed to be enabled even if
        # it is not explicitly specified or if auto_enable_ha says so.
        # Note that None is used to indicate that request did not include any
        # ha information was provided!
        requested_ha_enabled = router.pop(
            ha.ENABLED, True if requested_ha_details or auto_enable_ha is True
            else None)
        res = {}
        ha_currently_enabled = current_router.get(ha.ENABLED, False)
        # Note: must check for 'is True' as None implies attribute not given
        if requested_ha_enabled is True or ha_currently_enabled is True:
            if not cfg.CONF.ha.ha_support_enabled:
                raise ha.HADisabled()
            curr_ha_details = current_router.get(ha.DETAILS, {})
            if ha.TYPE in requested_ha_details:
                requested_ha_type = requested_ha_details[ha.TYPE]
                if (ha.TYPE in curr_ha_details and
                        requested_ha_type != curr_ha_details[ha.TYPE]):
                    raise ha.HATypeCannotBeChanged()
                elif requested_ha_type in cfg.CONF.ha.disabled_ha_mechanisms:
                    raise ha.HADisabledHAType(ha_type=requested_ha_type)
        if requested_ha_enabled:
            res[ha.ENABLED] = requested_ha_enabled
            if requested_ha_details:
                res[ha.DETAILS] = requested_ha_details
        elif requested_ha_enabled is False:
            res[ha.ENABLED] = False
        return res

    def _teardown_redundancy_router_gw_connectivity(self, context, router,
                                                    router_db,
                                                    plugging_driver):
        """To be called in update_router() if the router gateway is to change
        BEFORE router has been updated in DB .
        """
        if not router[ha.ENABLED]:
            # No HA currently enabled so we're done
            return
        e_context = context.elevated()
        # since gateway is about to change the ha group for the current gateway
        # is removed, a new one will be created later
        self._delete_ha_group(e_context, router_db.gw_port_id)
        # teardown connectivity for the gw ports on the redundancy routers
        # and remove those ports as new ones will be created later
        rr_ids = []
        for r_b_db in router_db.redundancy_bindings:
            if plugging_driver is not None:
                plugging_driver.teardown_logical_port_connectivity(
                    e_context, r_b_db.redundancy_router.gw_port,
                    r_b_db.redundancy_router.hosting_info.hosting_device_id)
            self._update_router_no_notify(
                e_context, r_b_db.redundancy_router_id,
                {'router': {EXTERNAL_GW_INFO: None, ha.ENABLED: False}})
            rr_ids.append(r_b_db.redundancy_router_id)
        self.notify_routers_updated(e_context, rr_ids)

    def _update_redundancy_routers(self, context, updated_router,
                                   update_specification, requested_ha_settings,
                                   updated_router_db, gateway_changed):
        """To be called in update_router() AFTER router has been
        updated in DB.
        """
        router_requested = update_specification['router']
        ha_settings_db = updated_router_db.ha_settings
        ha_enabled_requested = requested_ha_settings.get(ha.ENABLED, False)
        if not (updated_router[ha.ENABLED] or ha_enabled_requested):
            # No HA currently enabled and no HA requested so we're done
            return
        # The redundancy routers need interfaces on the same networks as the
        # user visible router.
        ports = self._get_router_interfaces(updated_router_db)
        e_context = context.elevated()
        if not updated_router[ha.ENABLED] and ha_enabled_requested:
            # No HA currently enabled but HA requested
            router_requested.update(requested_ha_settings)
            router_requested[EXTERNAL_GW_INFO] = (
                updated_router[EXTERNAL_GW_INFO])
            requested_ha_settings = self._ensure_create_ha_compliant(
                router_requested, updated_router_db.hosting_info.router_type)
            self._create_redundancy_routers(
                e_context, updated_router, requested_ha_settings,
                updated_router_db, ports, expire_db=True)
            return
        rr_ids = self._get_redundancy_router_ids(context, updated_router['id'])
        ha_details_update_spec = requested_ha_settings.get(ha.DETAILS)
        if (updated_router[ha.ENABLED] and not requested_ha_settings.get(
                ha.ENABLED, updated_router[ha.ENABLED])):
            # HA currently enabled but HA disable requested
            # delete ha settings and extra port for gateway (VIP) port
            self._delete_ha_group(e_context, updated_router_db.gw_port_id)
            self._remove_redundancy_routers(e_context, rr_ids, ports, True)
            with context.session.begin(subtransactions=True):
                context.session.delete(ha_settings_db)
        elif ha_details_update_spec:
            # HA currently enabled and HA setting update (other than
            # disable HA) requested
            old_redundancy_level = ha_settings_db.redundancy_level
            ha_settings_db.update(ha_details_update_spec)
            diff = (ha_details_update_spec.get(ha.REDUNDANCY_LEVEL,
                                               old_redundancy_level) -
                    old_redundancy_level)
            with context.session.begin(subtransactions=True):
                context.session.add(ha_settings_db)
            if diff < 0:
                # Remove -diff redundancy routers
                #TODO(bobmel): Ensure currently active router is excluded
                to_remove = rr_ids[len(rr_ids) + diff:]
                rr_ids = rr_ids[:len(rr_ids) + diff]
                self._remove_redundancy_routers(e_context, to_remove, ports)
            elif diff > 0:
                # Add diff redundancy routers
                start = old_redundancy_level + 1
                stop = start + diff
                self._add_redundancy_routers(e_context, start, stop,
                                             updated_router, ports,
                                             ha_settings_db, False)
            if gateway_changed is True:
                self._change_ha_for_gateway(e_context, updated_router,
                                            updated_router_db, ha_settings_db,
                                            router_requested, expire=True)
            else:
                # Notify redundancy routers about changes
                self.notify_routers_updated(e_context, rr_ids)

        elif gateway_changed is True:
            # HA currently enabled (and to remain so) nor any HA setting update
            # and gateway has changed
            self._change_ha_for_gateway(e_context, updated_router,
                                        updated_router_db, ha_settings_db,
                                        router_requested)
        # pick up updates to other attributes where it makes sense
        # and push - right now it is only admin_state_up.
        other_updates_spec = {'router': {}}
        if 'admin_state_up' in update_specification['router']:
            other_updates_spec['router']['admin_state_up'] = (
                update_specification['router']['admin_state_up'])
        if 'name' in update_specification['router']:
            other_updates_spec['router']['name'] = (
                update_specification['router']['name'])
        if (other_updates_spec['router'] or
            'routes' in update_specification['router']):
            self._process_other_router_updates(e_context, updated_router_db,
                                               other_updates_spec)
        # Ensure we get latest state from DB
        context.session.expire(updated_router_db)
        self._extend_router_dict_ha(updated_router, updated_router_db)

    def _change_ha_for_gateway(self, context, router, router_db,
                               ha_settings_db, update_spec, expire=False):
        # generate ha settings and extra port for router gateway VIP
        if router_db.gw_port is None:
            # gateway was removed and since gw's of redundancy routers are
            # removed by _teardown_redundancy_router_gw_connectivity we're done
            return
        gw_port = self._core_plugin._make_port_dict(router_db.gw_port)
        self._create_ha_group(context, router, gw_port, ha_settings_db)
        if expire is True:
            context.session.expire(router_db)
        # Now add gw to redundancy routers
        rr_ids = []
        for r_b_db in router_db.redundancy_bindings:
            spec = {EXTERNAL_GW_INFO: copy.deepcopy(router[EXTERNAL_GW_INFO])}
            if spec[EXTERNAL_GW_INFO]['external_fixed_ips']:
                # Ensure ip addresses are not specified as they cannot be
                # same as visible router's ip addresses.
                for e_fixed_ip in spec[EXTERNAL_GW_INFO]['external_fixed_ips']:
                    e_fixed_ip.pop('ip_address', None)
            #spec[EXTERNAL_GW_INFO].pop('external_fixed_ips', None)
            spec[ha.ENABLED] = False
            self._update_router_no_notify(
                context, r_b_db.redundancy_router_id, {'router': spec})
            rr_ids.append(r_b_db.redundancy_router_id)
        self.notify_routers_updated(context, rr_ids)

    def _process_other_router_updates(self, context, router_db, update_spec):
        rr_ids = []
        new_name_stub = update_spec['router'].get('name')
        for r_b_db in router_db.redundancy_bindings:
            if update_spec['router']:
                if new_name_stub is not None:
                    idx = r_b_db.redundancy_router.name.split('_')[-1]
                    update_spec['router']['name'] = (
                        new_name_stub + REDUNDANCY_ROUTER_SUFFIX + idx)
                update_spec['router'][ha.ENABLED] = False
                self._update_router_no_notify(
                    context, r_b_db.redundancy_router_id, update_spec)
            rr_ids.append(r_b_db.redundancy_router_id)
        self.notify_routers_updated(context, rr_ids)

    def _add_redundancy_routers(self, context, start_index, stop_index,
                                user_visible_router, ports=None,
                                ha_settings_db=None, create_ha_group=True):
        """Creates a redundancy router and its interfaces on
        the specified subnets.
        """
        priority = (DEFAULT_MASTER_PRIORITY +
                    (start_index - 1) * PRIORITY_INCREASE_STEP)
        r = copy.deepcopy(user_visible_router)
        # No tenant_id so redundancy routers are hidden from user
        r['tenant_id'] = ''
        name = r['name']
        redundancy_r_ids = []
        for i in range(start_index, stop_index):
            del r['id']
            # We don't replicate the user visible router's routes, instead
            # they are populated to redundancy routers for get router(s) ops
            r.pop('routes', None)
            # Redundancy routers will never have a route spec themselves
            # The redundancy routers must have HA disabled
            r[ha.ENABLED] = False
            r['name'] = name + REDUNDANCY_ROUTER_SUFFIX + str(i)
            # set role so that purpose of this router can be easily determined
            r[routerrole.ROUTER_ROLE_ATTR] = ROUTER_ROLE_HA_REDUNDANCY
            gw_info = r[EXTERNAL_GW_INFO]
            if gw_info and gw_info['external_fixed_ips']:
                # Ensure ip addresses are not specified as they cannot be
                # same as visible router's ip addresses.
                for e_fixed_ip in gw_info['external_fixed_ips']:
                    e_fixed_ip.pop('ip_address', None)
            r = self.create_router(context, {'router': r})
            LOG.debug("Created redundancy router %(index)d with router id "
                      "%(r_id)s", {'index': i, 'r_id': r['id']})
            priority += PRIORITY_INCREASE_STEP
            r_b_b = RouterRedundancyBinding(
                redundancy_router_id=r['id'],
                priority=priority,
                user_router_id=user_visible_router['id'])
            context.session.add(r_b_b)
            redundancy_r_ids.append(r['id'])
        for port_db in ports or []:
            port = self._core_plugin._make_port_dict(port_db)
            self._add_redundancy_router_interfaces(
                context, user_visible_router, None, port,
                redundancy_r_ids, ha_settings_db, create_ha_group)

    def _remove_redundancy_routers(self, context, router_ids, ports,
                                   delete_ha_groups=False):
        """Deletes all interfaces of the specified redundancy routers
        and then the redundancy routers themselves.
        """
        subnets_info = [{'subnet_id': port['fixed_ips'][0]['subnet_id']}
                        for port in ports]
        for r_id in router_ids:
            for i in range(len(subnets_info)):
                self.remove_router_interface(context, r_id, subnets_info[i])
                LOG.debug("Removed interface on %(s_id)s to redundancy router "
                          "with %(r_id)s",
                          {'s_id': ports[i]['network_id'], 'r_id': r_id})
                # There is only one ha group per network so only delete once
                if delete_ha_groups and r_id == router_ids[0]:
                    self._delete_ha_group(context, ports[i]['id'])
            self.delete_router(context, r_id)
            LOG.debug("Deleted redundancy router %s", r_id)

    def _get_router_interfaces(self, router_db,
                               port_type=DEVICE_OWNER_ROUTER_INTF):
        return [p['port'] for p in router_db.attached_ports if
                p['port_type'] == port_type]

    def _delete_redundancy_routers(self, context, router_db):
        """To be called in delete_router() BEFORE router has been
        deleted in DB. The router should have not interfaces.
        """
        e_context = context.elevated()
        for binding in router_db.redundancy_bindings:
            self.delete_router(e_context, binding.redundancy_router_id)
            LOG.debug("Deleted redundancy router %s",
                      binding.redundancy_router_id)
        if router_db.gw_port_id:
            # delete ha settings and extra port for gateway (VIP) port
            self._delete_ha_group(e_context, router_db.gw_port_id)

    def _add_redundancy_router_interfaces(self, context, router, itfc_info,
                                          new_port, redundancy_router_ids=None,
                                          ha_settings_db=None,
                                          create_ha_group=True):
        """To be called in add_router_interface() AFTER interface has been
        added to router in DB.
        """
        # There are essentially three cases where we add interface to a
        # redundancy router:
        # 1. HA is enabled on a user visible router that has one or more
        #    interfaces.
        # 2. Redundancy level is increased so one or more redundancy routers
        #    are added.
        # 3. An interface is added to a user visible router.
        #
        # For 1: An HA GROUP MUST BE CREATED and EXTRA PORTS MUST BE CREATED
        #        for each redundancy router. The id of extra port should be
        #        specified in the interface_info argument of the
        #        add_router_interface call so that we ADD BY PORT.
        # For 2: HA group need NOT be created as it will already exist (since
        #        there is already at least on redundancy router). EXTRA PORTS
        #        MUST BE CREATED for each added redundancy router. The id
        #        of extra port should be specified in the interface_info
        #        argument of the add_router_interface call so that we ADD BY
        #        PORT.
        # For 3: if the interface for the user_visible_router was added by ...
        #   a) PORT:   An HA GROUP MUST BE CREATED and and EXTRA PORTS MUST BE
        #              CREATED for each redundancy router. The id of extra port
        #              should be specified in the interface_info argument of
        #              the add_router_interface call so that we ADD BY PORT.
        #   b) SUBNET: There are two cases to consider. If the added interface
        #              of the user_visible_router has ...
        #        b1) 1 SUBNET:   An HA GROUP MUST BE CREATED and and EXTRA
        #                        PORTS MUST BE CREATED for each redundancy
        #                        router. The id of extra port should be
        #                        specified in the interface_info argument of
        #                        the add_router_interface call so we ADD BY
        #                        PORT.
        #        b2) >1 SUBNETS: HA group need NOT be created as it will
        #                        already exist (since the redundancy routers
        #                        should already have extra ports to which the
        #                        (IPv6) subnet is added. Extra ports need
        #                        thus NOT be created. The subnet id should be
        #                        added to the existing extra ports.
        router_id = router['id']
        if ha_settings_db is None:
            ha_settings_db = self._get_ha_settings_by_router_id(context,
                                                                router_id)
        if ha_settings_db is None:
            return
        e_context = context.elevated()
        add_by_subnet = (itfc_info is not None and 'subnet_id' in itfc_info and
                         len(new_port['fixed_ips']) > 1)
        if (add_by_subnet is False or (itfc_info is None and
                                       create_ha_group is True)):
            # generate ha settings and extra port for router (VIP) port
            self._create_ha_group(e_context, router, new_port, ha_settings_db)
        fixed_ips = self._get_fixed_ips_subnets(new_port['fixed_ips'])
        for r_id in (redundancy_router_ids or
                     self._get_redundancy_router_ids(e_context, router_id)):
            if add_by_subnet is True:
                # need to add subnet to redundancy router port
                ports = self._core_plugin.get_ports(
                    e_context,
                    filters={'device_id': [r_id],
                             'network_id': [new_port['network_id']]},
                    fields=['fixed_ips', 'id'])
                redundancy_port = ports[0]
                fixed_ips = redundancy_port['fixed_ips']
                fixed_ip = {'subnet_id': itfc_info['subnet_id']}
                fixed_ips.append(fixed_ip)
                self._core_plugin.update_port(
                    e_context, redundancy_port['id'],
                    {'port': {'fixed_ips': fixed_ips}})
            else:
                redundancy_port = self._create_hidden_port(
                    e_context, new_port['network_id'], '', fixed_ips)
                interface_info = {'port_id': redundancy_port['id']}
                self.add_router_interface(e_context, r_id, interface_info)

    def _update_redundancy_router_interfaces(self, context, router,
                                             port, modified_port_data,
                                             redundancy_router_ids=None,
                                             ha_settings_db=None):
        """To be called when the router interfaces are updated,
        like in the case of change in port admin_state_up status
        """
        router_id = router['id']
        if ha_settings_db is None:
            ha_settings_db = self._get_ha_settings_by_router_id(context,
                                                                router_id)
        if ha_settings_db is None:
            return
        e_context = context.elevated()

        rr_ids = self._get_redundancy_router_ids(e_context, router_id)
        port_info_list = self._core_plugin.get_ports(
            e_context, filters={'device_id': rr_ids,
                                'network_id': [port['network_id']]},
            fields=['device_id', 'id'])
        for port_info in port_info_list:
            self._core_plugin.update_port(e_context, port_info['id'],
                                          modified_port_data)
        self._update_hidden_port(e_context, port['id'], modified_port_data)

    def _create_ha_group(self, context, router, port, ha_settings_db):
        driver = self._get_router_type_driver(context,
                                              router[routertype.TYPE_ATTR])
        if driver is None:
            return
        ha_group_uuid = uuidutils.generate_uuid()
        # use HA group as device instead of the router to hide this port
        group_id = (driver.generate_ha_group_id(context, router, port,
                                                ha_settings_db,
                                                ha_group_uuid) or
                    self._generate_ha_group_id(context, router, port,
                                               ha_settings_db,
                                               ha_group_uuid))
        timers_cfg = driver.get_ha_group_timers_parameters(
            context, router, port, ha_settings_db, ha_group_uuid)
        tracking_cfg = driver.get_ha_group_tracking_parameters(
            context, router, port, ha_settings_db, ha_group_uuid)
        other_cfg = driver.get_other_ha_group_parameters(
            context, router, port, ha_settings_db, ha_group_uuid)
        if driver.ha_interface_ip_address_needed(
                context, router, port, ha_settings_db, ha_group_uuid):
            fixed_ips = self._get_fixed_ips_subnets(port['fixed_ips'])
            extra_port = self._create_hidden_port(
                context, port['network_id'], ha_group_uuid,
                fixed_ips, port['device_owner'])
            extra_port_id = extra_port['id']
        else:
            extra_port_id = None
        subnet_id = (port['fixed_ips'][0]['subnet_id']
                     if port['fixed_ips'] else None)

        with context.session.begin(subtransactions=True):
            r_ha_g = RouterHAGroup(
                id=ha_group_uuid,
                tenant_id=port['tenant_id'],
                ha_type=ha_settings_db.ha_type,
                group_identity=group_id,
                ha_port_id=port['id'],
                extra_port_id=extra_port_id,
                subnet_id=subnet_id,
                user_router_id=router['id'],
                timers_config=timers_cfg,
                tracking_config=tracking_cfg,
                other_config=other_cfg)
            context.session.add(r_ha_g)
        return r_ha_g

    def _get_fixed_ips_subnets(self, fixed_ips):
        subnets = copy.deepcopy(fixed_ips)
        for s in subnets:
            s.pop('ip_address', None)
        return subnets

    def _remove_redundancy_router_interfaces(self, context, router_id,
                                             old_port):
        """To be called in delete_router_interface() BEFORE interface has been
        removed from router in DB.
        """
        ha_settings = self._get_ha_settings_by_router_id(context, router_id)
        if ha_settings is None or old_port is None:
            return
        e_context = context.elevated()
        rr_ids = self._get_redundancy_router_ids(e_context, router_id)
        port_info_list = self._core_plugin.get_ports(
            e_context, filters={'device_id': rr_ids,
                                'network_id': [old_port['network_id']]},
            fields=['device_id', 'fixed_ips', 'id'])
        subnet_id = old_port['fixed_ips'][0]['subnet_id']
        for port_info in port_info_list:
            if port_info['fixed_ips'][0]['subnet_id'] == subnet_id:
                interface_info = {'port_id': port_info['id']}
                self.remove_router_interface(e_context, port_info['device_id'],
                                             interface_info)
        self._delete_ha_group(e_context, old_port['id'])

    def _redundancy_routers_for_floatingip(
            self, context, router_id, redundancy_router_ids=None,
            ha_settings_db=None):
        """To be called in update_floatingip() to get the
            redundant router ids.
        """
        if ha_settings_db is None:
            ha_settings_db = self._get_ha_settings_by_router_id(context,
                                                                router_id)
        if ha_settings_db is None:
            return

        e_context = context.elevated()
        router_ids = []
        for r_id in (redundancy_router_ids or
                     self._get_redundancy_router_ids(e_context, router_id)):
            router_ids.append(r_id)

        return router_ids

    def _update_hidden_port(self, context, ha_port_id, modified_port_data):
        hag = self._get_ha_group_by_ha_port_id(context, ha_port_id)
        if hag is not None:
            self._core_plugin.update_port(context, hag.extra_port_id,
                                          modified_port_data)

    def _delete_ha_group(self, context, ha_port_id):
        hag = self._get_ha_group_by_ha_port_id(context, ha_port_id)
        if hag is not None and hag.extra_port_id is not None:
            self._core_plugin.delete_port(context, hag.extra_port_id,
                                          l3_port_check=False)
            with context.session.begin(subtransactions=True):
                context.session.delete(hag)

    @staticmethod
    @bc.extends([l3_const.ROUTERS])
    def _extend_router_dict_ha(router_res, router_db):
        self = bc.get_plugin(bc.constants.L3)
        if extensions.is_extension_supported(self, ha.HA_ALIAS):
            ha_s = router_db.ha_settings
            rr_b = router_db.redundancy_binding
            if rr_b and rr_b.user_router:
                # include static routes from user visible router
                temp = {}
                self._extend_router_dict_extraroute(temp,
                                                    rr_b.user_router)
                if temp['routes']:
                    router_res['routes'].extend(temp['routes'])
            router_res[ha.ENABLED] = False if ha_s is None else True
            if router_res[ha.ENABLED]:
                ha_details = {ha.TYPE: ha_s.ha_type,
                              ha.PRIORITY: ha_s.priority,
                              ha.STATE: ha_s.state,
                              ha.REDUNDANCY_LEVEL: ha_s.redundancy_level,
                              ha.PROBE_CONNECTIVITY: ha_s.probe_connectivity}
                if ha_details[ha.PROBE_CONNECTIVITY]:
                    ha_details.update({ha.PROBE_TARGET: ha_s.probe_target,
                                       ha.PROBE_INTERVAL: ha_s.probe_interval})
                ha_details[ha.REDUNDANCY_ROUTERS] = (
                    [{'id': b.redundancy_router_id, ha.PRIORITY: b.priority,
                      ha.STATE: b.state}
                     for b in router_db.redundancy_bindings])
                router_res[ha.DETAILS] = ha_details
            else:
                # ensure any router details are removed
                router_res.pop(ha.DETAILS, None)

    def _populate_ha_information(self, context, router):
        """To be called when router information, including router interface
        list, (for the l3_cfg_agent) has been collected so it is extended
        with ha information.
        """
        r_r_b = self._get_redundancy_router_bindings(
            context, redundancy_router_id=router['id'])
        if not r_r_b:
            if router[ha.ENABLED]:
                # The router is a user visible router with HA enabled.
                user_router_id = router['id']
                fips = []
            else:
                # The router is a user visible router with HA disabled.
                # Nothing more to do here.
                return
        else:
            # The router is a redundancy router.
            # Need to fetch floatingip configurations from user visible router
            # so they can be added to the redundancy routers.
            user_router_id = r_r_b[0].user_router_id
            fips = self.get_floatingips(context,
                                        {'router_id': [user_router_id]})
        if router['id'] != user_router_id:
            # We add the HA settings from user visible router to
            # its redundancy routers.
            user_router_db = self._get_router(context, user_router_id)
            self._extend_router_dict_ha(router, user_router_db)
        # The interfaces of the user visible router must use the
        # IP configuration of the extra ports in the HA groups.
        hag_dbs = self._get_subnet_id_indexed_ha_groups(context,
                                                        user_router_id)
        e_context = context.elevated()
        if router.get('gw_port'):
            modified_interfaces = []
            interface_port = self._populate_port_ha_information(
                e_context, router['gw_port'], router['id'], hag_dbs,
                user_router_id, modified_interfaces)
            if not interface_port:
                # The router has a gw_port but cannot find the port info yet
                # so mark this router to have incomplete info and bail.
                # The cfg_agent puts this in the updated_routers to ask again.
                router['status'] = cisco_constants.ROUTER_INFO_INCOMPLETE
                return
            if modified_interfaces:
                router['gw_port'] = interface_port
        modified_interfaces = []
        for itfc in router.get(bc.constants.INTERFACE_KEY, []):
            interface_port = self._populate_port_ha_information(
                e_context, itfc, router['id'], hag_dbs, user_router_id,
                modified_interfaces)
            if not interface_port:
                # the router has interfaces but cannot find the port info yet
                # so mark this router to have incomplete info and bail
                # the cfg_agent will put this in the updated_list to ask again
                router['status'] = cisco_constants.ROUTER_INFO_INCOMPLETE
                return
        if modified_interfaces:
            router[bc.constants.INTERFACE_KEY] = modified_interfaces
        if fips:
            router[bc.constants.FLOATINGIP_KEY] = fips

    def _populate_port_ha_information(self, context, port, router_id, hag_dbs,
                                      user_router_id, modified_interfaces):
        subnet_id = port['fixed_ips'][0]['subnet_id']
        try:
            hag_db = hag_dbs[subnet_id]
        except KeyError:
            # Oops, the subnet_id was not found. Probably because the DB
            # insertion of that HA group is still in progress by another
            # process and has not been committed to the DB yet.
            # Let's retry a few times to see if the DB entry turns up.
            LOG.debug('No HA group info for router: %(r_id)s and subnet: '
                      '%(s_id)s was found when populating HA info for port: '
                      '%(p_id)s. Will now make additional lookup attempts.',
                      {'r_id': router_id, 's_id': subnet_id,
                       'p_id': port['id']})
            try:
                hag_db = self._get_ha_group_for_subnet_id(context, router_id,
                                                          subnet_id)
            except exc.NoResultFound:
                hag_db = None
        if hag_db is None:
            LOG.debug('Failed to fetch the HA group info for router: '
                      '%(r_id)s and subnet: %(s_id)s. Giving up. No HA '
                      'info will be added to the router\'s port: %(p_id)s.',
                      {'r_id': router_id, 's_id': subnet_id,
                       'p_id': port['id']})
            # we leave it to the L3 config agent to handle this
            return
        else:
            LOG.debug('Successfully fetched the HA group info for '
                      'router: %(r_id)s and subnet: %(s_id)s from DB',
                      {'r_id': router_id, 's_id': subnet_id})
            hag_dbs[subnet_id] = hag_db
        if router_id == user_router_id:
            # If the router interface needs no dedicated IP address we just
            # set the HA (VIP) port to the port itself. The config agent
            # driver will know how to handle this "signal".
            p_id = hag_db.extra_port_id or port['id']
            try:
                interface_port = self._core_plugin.get_port(context, p_id)
            except n_exc.PortNotFound:
                LOG.debug('Failed to get port %(p_id)s from DB for user '
                          'visible HA router: %(r_id)s. No HA info will be '
                          'added to that router\'s port', {'p_id': p_id,
                                                           'r_id': router_id})
                return
            LOG.debug('Fetched port %(p_id)s from DB for user visible '
                      'router: %(r_id)s', {'p_id': p_id, 'r_id': router_id})
            self._populate_mtu_and_subnets_for_ports(context, [interface_port])
            modified_interfaces.append(interface_port)
            ha_port = port
        else:
            try:
                ha_port = self._core_plugin.get_port(context,
                                                     hag_db.ha_port_id)
            except n_exc.PortNotFound:
                LOG.debug('Failed to get port %(p_id)s from DB for HA '
                          'redundancy router: %(r_id)s. No HA info will be '
                          'added to that router\'s port',
                          {'p_id': hag_db.ha_port_id, 'r_id': router_id})
                return
            LOG.debug('Fetched port %(p_id)s from DB for HA redundancy '
                      'router: %(r_id)s', {'p_id': hag_db.ha_port_id,
                                           'r_id': router_id})
            self._populate_mtu_and_subnets_for_ports(context, [ha_port])
            interface_port = port
        interface_port[ha.HA_INFO] = {
            ha.TYPE: hag_db.ha_type,
            HA_GROUP: hag_db.group_identity,
            'timers_config': hag_db.timers_config,
            'tracking_config': hag_db.tracking_config,
            'other_config': hag_db.other_config,
            HA_PORT: ha_port}
        return interface_port

    def _create_hidden_port(self, context, network_id, device_id, fixed_ips,
                            port_type=DEVICE_OWNER_ROUTER_INTF):
        """Creates port used specially for HA purposes."""
        port = {'port': {
            'tenant_id': '',  # intentionally not set
            'network_id': network_id,
            'mac_address': ATTR_NOT_SPECIFIED,
            'fixed_ips': fixed_ips,
            'device_id': device_id,
            'device_owner': port_type,
            'admin_state_up': True,
            'name': ''}}
        if extensions.is_extension_supported(self._core_plugin,
                                             "dns-integration"):
            port['port'].update(dns_name='')
        core_plugin = bc.get_plugin()
        return core_plugin.create_port(context, port)

    def _get_ha_settings_by_router_id(self, context, router_id):
        query = context.session.query(RouterHASetting)
        query = query.filter(RouterHASetting.router_id == router_id)
        try:
            r_ha_s = query.one()
        except (exc.NoResultFound, exc.MultipleResultsFound):
            return
        return r_ha_s

    def _get_ha_group_by_ha_port_id(self, context, port_id):
        query = context.session.query(RouterHAGroup)
        query = query.filter(RouterHAGroup.ha_port_id == port_id)
        try:
            r_ha_g = query.one()
        except (exc.NoResultFound, exc.MultipleResultsFound):
            return
        return r_ha_g

    def _get_subnet_id_indexed_ha_groups(self, context, router_id,
                                         load_virtual_port=False):
        query = context.session.query(RouterHAGroup)
        query = query.filter(RouterHAGroup.user_router_id == router_id)
        if load_virtual_port:
            query = query.options(joinedload('redundancy_router'))
        return {hag['subnet_id']: hag for hag in query}

    @cisco_utils.retry(exc.NoResultFound, LOOKUP_RETRIES, RETRY_INTERVAL, 1)
    def _get_ha_group_for_subnet_id(self, context, router_id, subnet_id):
        query = context.session.query(RouterHAGroup)
        query = query.filter_by(user_router_id=router_id,
                                subnet_id=subnet_id)
        LOG.debug('Trying to fetch HA group info for router: %(r_id)s and '
                  'subnet: %(s_id)s', {'r_id': router_id,
                                       's_id': subnet_id})
        return query.one()

    def _get_redundancy_router_bindings(self, context, router_id=None,
                                        redundancy_router_id=None):
        query = context.session.query(RouterRedundancyBinding)
        if router_id is not None:
            query = query.filter(
                RouterRedundancyBinding.user_router_id == router_id)
        if redundancy_router_id is not None:
            query = query.filter(
                RouterRedundancyBinding.redundancy_router_id ==
                redundancy_router_id)
        query = query.order_by(RouterRedundancyBinding.priority)
        return query.all()

    def _get_redundancy_router_ids(self, context, router_id):
        return [binding.redundancy_router_id for binding in
                self._get_redundancy_router_bindings(context,
                                                     router_id=router_id)]

    def _generate_ha_group_id(self, context, router, port, ha_settings_db,
                              ha_group_uuid):
        #TODO(bob-melander): Generate "guaranteed" unique id
        if ha_settings_db.ha_type == ha.HA_HSRP:
            return random.randint(0, MAX_HSRP_GROUPS)
        elif ha_settings_db.ha_type == ha.HA_VRRP:
            return random.randint(0, MAX_VRRP_GROUPS)
        else:
            # ha_type must be ha_type.GLBP
            return random.randint(0, MAX_GLBP_GROUPS)

    # Overloaded function from l3_db
    def get_router_for_floatingip(self, context, internal_port,
                                  internal_subnet, external_network_id):
        """We need to over-load this function so that we only return the
        user visible router and never its redundancy routers (as they never
        have floatingips associated with them).
        """
        gw_port = orm.aliased(models_v2.Port, name="gw_port")
        routerport_qry = context.session.query(
            RouterPort.router_id, models_v2.IPAllocation.ip_address).join(
            models_v2.Port, models_v2.IPAllocation).filter(
            models_v2.Port.network_id == internal_port['network_id'],
            RouterPort.port_type.in_(bc.constants.ROUTER_INTERFACE_OWNERS),
            models_v2.IPAllocation.subnet_id == internal_subnet['id']
        ).join(gw_port, gw_port.device_id == RouterPort.router_id).filter(
            gw_port.network_id == external_network_id,
            gw_port.device_owner == bc.constants.DEVICE_OWNER_ROUTER_GW
        ).distinct()

        # Ensure that redundancy routers (in a ha group) are not returned,
        # since only the user visible router should have floatingips.
        # This can be done by checking that the id of routers does not
        # appear in the 'redundancy_router_id' column in the
        # 'cisco_router_redundancy_bindings' table.
        routerport_qry = routerport_qry.outerjoin(
            RouterRedundancyBinding,
            RouterRedundancyBinding.redundancy_router_id ==
            RouterPort.router_id)
        routerport_qry = routerport_qry.filter(
            RouterRedundancyBinding.redundancy_router_id == expr.null())

        first_router_id = None
        for router_id, interface_ip in routerport_qry:
            if interface_ip == internal_subnet['gateway_ip']:
                return router_id
            if not first_router_id:
                first_router_id = router_id
        if first_router_id:
            return first_router_id

        raise l3_exceptions.ExternalGatewayForFloatingIPNotFound(
            subnet_id=internal_subnet['id'],
            external_network_id=external_network_id,
            port_id=internal_port['id'])
