# encoding: utf-8
import concurrent.futures
import itertools
import datetime
import decimal
import logging
import random
import re
import uuid
from string import ascii_letters as letters
from unittest import (
    skip,
    skipIf,
)

# LIBRARIES
import sleuth
from django.conf import settings
from django.core.cache import cache
from django.core.exceptions import (
    FieldError,
    ValidationError,
)
from django.db import (
    DatabaseError,
    DataError,
    IntegrityError,
    NotSupportedError,
    transaction,
)
from django.db import connection as default_connection
from django.db import connections
from django.db import models
from django.db.models.query import Q
from django.forms import ModelForm
from django.forms.models import modelformset_factory
from django.test import RequestFactory
from django.test.utils import override_settings
from django.urls import path
from django.utils.safestring import SafeText

from django.utils.timezone import make_aware
from datetime import timezone as tz
from gcloudc.db.backends.common.base.entity import Entity, Key

from gcloudc.db.backends.common import indexing
from gcloudc.db.backends.common.parsers.expressions import Scatter

from gcloudc.db.backends.common.commands import FlushCommand
from gcloudc.db.backends.common.indexing import (
    IExactIndexer,
    add_special_index,
    get_indexer,
)
from gcloudc.db.backends.common.mixins import UniquenessMixin
from gcloudc.db.backends.common.constraints import (
    _unique_combinations,
    unique_identifiers_from_entity,
)
from gcloudc.db.backends.common.helpers import (
    decimal_to_string,
    entity_matches_query,
    normalise_field_value,
)

from . import TestCase, firestore_only, datastore_only
from .models import (
    Animal,
    DateTimeModel,
    DurationModel,
    Enclosure,
    IntegerModel,
    ManyFields,
    ModelWithDates,
    ModelWithNullableCharField,
    ModelWithUniques,
    ModelWithUniquesOnForeignKey,
    MultiTableChildOne,
    MultiTableChildTwo,
    MultiTableParent,
    NullDate,
    NullDateSet,
    NullUUID,
    Permission,
    SelfRelatedModel,
    SpecialIndexesModel,
    TestFruit,
    TestUser,
    UniqueModel,
    UniqueModelWithLongPK,
    UUIDTestModel,
    Zoo,
)

DEFAULT_NAMESPACE = default_connection.ops.connection.settings_dict.get("NAMESPACE")


def set_test_many_fields_model_field(model, field_name, field_value):
    setattr(model, field_name, field_value)
    model.save(update_fields=[field_name])


class BackendTests(TestCase):
    def test_pk_gt_empty_returns_all(self):
        for i in range(10):
            TestFruit.objects.create(name=str(i), color=str(i))

        self.assertEqual(10, TestFruit.objects.filter(pk__gt="").count())
        self.assertEqual(10, TestFruit.objects.filter(pk__gte="").count())
        self.assertEqual(0, TestFruit.objects.filter(pk__lt="").count())
        self.assertEqual(0, TestFruit.objects.filter(pk__lte="").count())

    def test_pk_gt_zero_returns_all(self):
        IntegerModel.objects.create(pk=1, integer_field=1)
        IntegerModel.objects.create(pk=2, integer_field=2)

        results = IntegerModel.objects.filter(pk__gt=0)
        self.assertEqual(2, len(results))

        results = IntegerModel.objects.filter(pk__gte=0)
        self.assertEqual(2, len(results))

        results = IntegerModel.objects.filter(pk__lt=0)
        self.assertEqual(0, len(results))

        results = IntegerModel.objects.filter(pk__lte=0)
        self.assertEqual(0, len(results))

    def test_entity_matches_query(self):
        k = default_connection.connection.new_key(
            "test_model", 1, namespace=default_connection.namespace
        )
        entity = Entity(k)
        entity["name"] = "Charlie"
        entity["age"] = 22

        query = default_connection.get_query_class()(
            table_or_path=["test_model"], namespace=default_connection.namespace,
            connection=default_connection
        )
        query.add_filter("name", "=", "Charlie")
        self.assertTrue(entity_matches_query(entity, query))

        query = default_connection.get_query_class()(
            table_or_path=["test_model"], namespace=default_connection.namespace,
            connection=default_connection
        )
        query.add_filter("age", ">=", 5)
        self.assertTrue(entity_matches_query(entity, query))

        query = default_connection.get_query_class()(
            table_or_path=["test_model"], namespace=default_connection.namespace,
            connection=default_connection
        )
        query.add_filter("age", "<", 22)
        self.assertFalse(entity_matches_query(entity, query))

        query = default_connection.get_query_class()(
            table_or_path=["test_model"], namespace=default_connection.namespace,
            connection=default_connection
        )
        query.add_filter("age", "<=", 22)
        self.assertTrue(entity_matches_query(entity, query))

        query = default_connection.get_query_class()(
            table_or_path=["test_model"], namespace=default_connection.namespace,
            connection=default_connection
        )
        query.add_filter("name", "=", "Fred")
        self.assertFalse(entity_matches_query(entity, query))

        query = default_connection.get_query_class()(
            table_or_path=["test_model"], namespace=default_connection.namespace,
            connection=default_connection
        )
        query.add_filter("name", "array_contains", "Fred")
        # If the entity has a list field, then if any of them match the
        # query then it's a match
        entity["name"] = ["Bob", "Fred", "Dave"]
        self.assertTrue(entity_matches_query(entity, query))  # ListField test

        entity["name"] = None
        query.add_filter("name", "array_contains", "Fred")
        self.assertFalse(entity_matches_query(entity, query))  # ListField test

    def test_entity_matches_query_same_field_different_collection(self):
        """
            Test that the entity_matches_query does not match entities from
            a different collection.
        """
        k = default_connection.connection.new_key(
            "test_model", 1, namespace=default_connection.namespace
        )
        entity = Entity(k)
        entity["name"] = "Charlie"
        entity["age"] = 22

        query = default_connection.get_query_class()(
            table_or_path=["another_model"], namespace=default_connection.namespace,
            connection=default_connection
        )
        query.add_filter("name", "=", "Charlie")
        self.assertFalse(entity_matches_query(entity, query))

    def test_entity_matches_query_array_contains_all(self):
        """
            Test that the entity_matches_query works with array_contains_all operator.
        """
        k = default_connection.connection.new_key(
            "test_model", 1, namespace=default_connection.namespace
        )
        entity = Entity(k)
        entity["names"] = ["Charlie", "Fred", "Dave"]

        query = default_connection.get_query_class()(
            table_or_path=["test_model"], namespace=default_connection.namespace,
            connection=default_connection
        )

        query.add_filter("names", "array_contains_all", ["Charlie", "Fred"])
        self.assertTrue(entity_matches_query(entity, query))

        query.add_filter("names", "array_contains_all", ["Charlie", "Another"])
        self.assertFalse(entity_matches_query(entity, query))

        entity["names"] = None

        query.add_filter("names", "array_contains_all", ["Charlie", "Fred"])
        self.assertFalse(entity_matches_query(entity, query))

    def test_exclude_pks_with_slice(self):
        for i in range(10):
            TestFruit.objects.create(name=str(i), color=str(i))

        to_exclude = [str(x) for x in list(range(5)) + list(range(15, 20))]

        to_return = TestFruit.objects.exclude(pk__in=set(to_exclude)).values_list("pk", flat=True)[:2]
        self.assertEqual(2, len(to_return))

        qs = TestFruit.objects.filter(
            pk__in=to_return
        )

        self.assertEqual(2, len(qs))

    def test_count_on_excluded_pks(self):
        TestFruit.objects.create(name="Apple", color="Red")
        TestFruit.objects.create(name="Orange", color="Orange")

        self.assertEqual(1, TestFruit.objects.filter(pk__in=["Apple", "Orange"]).exclude(pk__in=["Apple"]).count())

    def test_defaults(self):
        fruit = TestFruit.objects.create(name="Apple", color="Red")
        self.assertEqual("Unknown", fruit.origin)

        rpc = default_connection.connection

        instance = rpc.get(
            default_connection.connection.new_key(
                TestFruit._meta.db_table, fruit.pk, namespace=default_connection.namespace
            )
        )

        del instance["origin"]
        rpc.put(instance)

        fruit = TestFruit.objects.get()
        self.assertIsNone(fruit.origin)
        fruit.save()

        fruit = TestFruit.objects.get()
        self.assertEqual("Unknown", fruit.origin)

    def test_get_by_keys(self):
        colors = ["Red", "Green", "Blue", "Yellow", "Orange"]
        fruits = [TestFruit.objects.create(name=str(x), color=random.choice(colors)) for x in range(32)]

        keys = [str(x) for x in range(32)]
        results = list(TestFruit.objects.only("color").filter(pk__in=keys).order_by("name"))  # One multiquery

        # Confirm the ordering is correct
        self.assertEqual(sorted(keys), [x.pk for x in results])

        results = list(TestFruit.objects.only("color").filter(pk__in=keys).order_by("name")[5:10])
        self.assertEqual(len(results), 5)
        self.assertEqual([x.pk for x in results], sorted(keys)[5:10])

        # Make sure we can do a normal (non-projection) get by keys
        self.assertItemsEqual(TestFruit.objects.filter(pk__in=keys), fruits)

    def test_get_or_create(self):
        """
            Django's get_or_create can do the following:

            1. get(**lookup) -> throws DoesNotExist
            2. Catches DoesNotExist
            3. create() -> throws IntegrityError
            4. get(**lookup)

            This test proves that we throw the right kind of error at step 3 when
            unique constraints are violated.
        """

        def wrap_get(func):
            def _wrapped(*args, **kwargs):
                try:
                    if _wrapped.calls == 0:
                        raise UniqueModel.DoesNotExist()
                    else:
                        return func(*args, **kwargs)
                finally:
                    _wrapped.calls += 1

            _wrapped.calls = 0
            return _wrapped

        from django.db.models import query
        wrapped_get = wrap_get(query.QuerySet.get)

        UniqueModel.objects.create(unique_field="Test")

        with sleuth.switch("django.db.models.query.QuerySet.get", wrapped_get):
            instance, created = UniqueModel.objects.get_or_create(unique_field="Test")
            self.assertFalse(created)

    def test_setting_non_null_null_throws_integrity_error(self):
        with self.assertRaises(IntegrityError):
            IntegerModel.objects.create(integer_field=None)

        with self.assertRaises(IntegrityError):
            instance = IntegerModel()
            instance.integer_field = None
            instance.save()

        with self.assertRaises(IntegrityError):
            instance = IntegerModel.objects.create(integer_field=1)
            instance = IntegerModel.objects.get()
            instance.integer_field = None
            instance.save()

    def test_normalise_field_value(self):
        self.assertEqual(u'0000475231073257', normalise_field_value(decimal.Decimal(475231073257)))
        self.assertEqual(u'-0000475231073257', normalise_field_value(decimal.Decimal(-475231073257)))
        self.assertEqual(u'0000000004752311', normalise_field_value(decimal.Decimal(4752310.73257)))
        self.assertEqual(u'0000004752310733', normalise_field_value(decimal.Decimal(4752310732.57)))
        self.assertEqual(
            datetime.datetime(2015, 1, 27, 2, 46, 8, 584258),
            normalise_field_value(datetime.datetime(2015, 1, 27, 2, 46, 8, 584258))
        )

    def test_decimal_to_string(self):
        self.assertEqual(u'0002312487812767', decimal_to_string(decimal.Decimal(2312487812767)))
        self.assertEqual(u'-0002312487812767', decimal_to_string(decimal.Decimal(-2312487812767)))
        self.assertEqual(u'002312487812', decimal_to_string(decimal.Decimal(2312487812), 12))
        self.assertEqual(u'002387812.320', decimal_to_string(decimal.Decimal(2387812.32), 12, 3))
        self.assertEqual(u'-002387812.513', decimal_to_string(decimal.Decimal(-2387812.513212), 12, 3))
        self.assertEqual(u'0237812.000', decimal_to_string(decimal.Decimal(237812), 10, 3))
        self.assertEqual(u'-0237812.210', decimal_to_string(decimal.Decimal(-237812.21), 10, 3))

    def test_gae_query_display(self):
        # Shouldn't raise any exceptions:
        representation = str(TestUser.objects.filter(username='test-value').query)
        self.assertTrue('test-value' in representation)
        self.assertTrue('username' in representation)

    def test_range_behaviour(self):
        IntegerModel.objects.create(integer_field=5)
        IntegerModel.objects.create(integer_field=10)
        IntegerModel.objects.create(integer_field=15)

        self.assertItemsEqual(
            [10],
            IntegerModel.objects.filter(
                integer_field__range=(6, 14)
            ).values_list("integer_field", flat=True)
        )

        self.assertItemsEqual(
            [5, 10, 15],
            IntegerModel.objects.filter(
                integer_field__range=(5, 15)
            ).order_by("integer_field").values_list("integer_field", flat=True)
        )

        self.assertItemsEqual(
            [5, 15],
            IntegerModel.objects.exclude(
                integer_field__range=(6, 14)
            ).values_list("integer_field", flat=True)
        )

    def test_exclude_nullable_field(self):
        instance = ModelWithNullableCharField.objects.create(some_id=999)  # Create a nullable thing
        ModelWithNullableCharField.objects.create(some_id=999, field1="test")  # Create a nullable thing
        self.assertItemsEqual(
            [instance],
            ModelWithNullableCharField.objects.filter(
                some_id=999
            ).exclude(field1="test").all()
        )

        instance.field1 = "bananas"
        instance.save()

        self.assertEqual(instance, ModelWithNullableCharField.objects.filter(some_id=999).exclude(field1="test")[0])

    def test_null_uuid_field(self):
        i1 = NullUUID.objects.create(id=1)
        i2 = NullUUID.objects.create(id=2)

        self.assertEqual(sorted([x.id for x in NullUUID.objects.all()]), [i1.pk, i2.pk])
        self.assertEqual(len(NullUUID.objects.values()), 2)
        self.assertEqual(len(NullUUID.objects.values_list()), 2)
        self.assertEqual(len(NullUUID.objects.filter(uuid__isnull=True)), 2)
        self.assertEqual([x.uuid for x in NullUUID.objects.all()], [None, None])

    def test_null_date_field(self):
        null_date = NullDate()
        null_date.save()

        null_date = NullDate.objects.get()
        self.assertIsNone(null_date.date)
        self.assertIsNone(null_date.time)
        self.assertIsNone(null_date.datetime)

    def test_convert_str_subclasses_to_str(self):
        # The App Engine SDK raises BadValueError if you try saving a SafeText
        # string to a CharField. Djangae explicitly converts it to str.
        grue = SafeText(u'grue')

        self.assertIsInstance(grue, str)
        self.assertNotEqual(type(grue), str)

        obj = TestFruit.objects.create(name=u'foo', color=grue)
        obj = TestFruit.objects.get(pk=obj.pk)
        self.assertEqual(type(obj.color), str)

        obj = TestFruit.objects.filter(color=grue)[0]
        self.assertEqual(type(obj.color), str)

    def test_notsupportederror_thrown_on_too_many_inequalities(self):
        TestFruit.objects.create(name="Apple", color="Green", origin="England")
        pear = TestFruit.objects.create(name="Pear", color="Green")
        banana = TestFruit.objects.create(name="Banana", color="Yellow")

        # Excluding one field is fine
        self.assertItemsEqual([pear, banana], list(TestFruit.objects.exclude(name="Apple")))

        # Excluding a field, and doing a > or < on another is not so fine
        with self.assertRaises(NotSupportedError):
            self.assertEqual(pear, TestFruit.objects.exclude(origin="England").filter(color__lt="Yellow").get())

        # Same with excluding two fields
        with self.assertRaises(NotSupportedError):
            list(TestFruit.objects.exclude(origin="England").exclude(color="Yellow"))

        # But apparently excluding the same field twice is OK
        self.assertItemsEqual(
            [banana],
            list(TestFruit.objects.exclude(
                origin="England"
            ).exclude(name="Pear").order_by("origin"))
        )

        # And apparently having both a __gt and a __lt filter on the same field is also fine
        self.assertItemsEqual([banana], list(TestFruit.objects.order_by().filter(name__lt="Pear", name__gt="Apple")))

    # FIXME: This is only skipped on Firestore because it orders by ID descending
    # and Firestore can't do that (:shrug:). If Firestore gains that ability, or
    # we find a workaround then we can enable this. Alternatively we can update
    # the test to not do a descending query.
    @datastore_only
    def test_excluding_pks_is_emulated(self):
        apple = TestFruit.objects.create(name="Apple", color="Green", is_mouldy=True, origin="England")
        banana = TestFruit.objects.create(name="Banana", color="Yellow", is_mouldy=True, origin="Dominican Republic")
        cherry = TestFruit.objects.create(name="Cherry", color="Red", is_mouldy=True, origin="Germany")
        pear = TestFruit.objects.create(name="Pear", color="Green", origin="England")

        self.assertEqual(
            [apple, pear],
            list(TestFruit.objects.filter(origin__lt="Germany").exclude(
                pk=banana.pk
            ).exclude(pk=cherry.pk).order_by("origin"))
        )

        self.assertEqual(
            [apple, cherry],
            list(TestFruit.objects.exclude(
                origin="Dominican Republic"
            ).exclude(pk=pear.pk).order_by("origin"))
        )

        self.assertEqual(
            [],
            list(TestFruit.objects.filter(
                is_mouldy=True
            ).filter(
                color="Green", origin__gt="England"
            ).exclude(pk=pear.pk).order_by("-origin"))
        )

        self.assertEqual([cherry, banana], list(TestFruit.objects.exclude(pk=pear.pk).order_by("-name")[:2]))
        self.assertEqual([banana, apple], list(TestFruit.objects.exclude(pk=pear.pk).order_by("origin", "name")[:2]))

    def test_datetime_fields(self):
        date = datetime.datetime.today()
        dt = datetime.datetime.now()
        time = datetime.time(0, 0, 0)

        # check if creating objects work
        obj = NullDate.objects.create(date=date, datetime=dt, time=time)

        # check if filtering objects work
        self.assertItemsEqual([obj], NullDate.objects.filter(datetime=dt))
        self.assertItemsEqual([obj], NullDate.objects.filter(date=date))
        self.assertItemsEqual([obj], NullDate.objects.filter(time=time))

        # check if updating objects work
        obj.date = date + datetime.timedelta(days=1)
        obj.datetime = dt + datetime.timedelta(days=1)
        obj.time = datetime.time(23, 0, 0)
        obj.save()
        self.assertItemsEqual([obj], NullDate.objects.filter(datetime=obj.datetime))
        self.assertItemsEqual([obj], NullDate.objects.filter(date=obj.date))
        self.assertItemsEqual([obj], NullDate.objects.filter(time=obj.time))

    def test_related_datetime_nullable(self):
        date = datetime.datetime.today()
        dt = datetime.datetime.now()
        time = datetime.time(0, 0, 0)

        date_set = NullDateSet.objects.create()
        empty_obj = NullDate.objects.create(date=None, datetime=None, time=None)
        date_set.dates.add(empty_obj)

        obj = NullDate.objects.create(date=date, datetime=dt, time=time)
        date_set.dates.add(obj)
        date_set.save()

        # check if filtering/excluding of None works in RelatedSetField
        self.assertItemsEqual([obj], date_set.dates.filter(datetime__isnull=False))
        self.assertItemsEqual([obj], date_set.dates.filter(date__isnull=False))
        self.assertItemsEqual([obj], date_set.dates.filter(time__isnull=False))

        self.assertItemsEqual([obj], date_set.dates.exclude(datetime=None))
        self.assertItemsEqual([obj], date_set.dates.exclude(date=None))
        self.assertItemsEqual([obj], date_set.dates.exclude(time=None))

        # sorting should work too
        self.assertItemsEqual([obj, empty_obj], date_set.dates.order_by('datetime'))
        self.assertItemsEqual([empty_obj, obj], date_set.dates.order_by('-datetime'))
        self.assertItemsEqual([obj, empty_obj], date_set.dates.order_by('date'))
        self.assertItemsEqual([empty_obj, obj], date_set.dates.order_by('-date'))
        self.assertItemsEqual([obj, empty_obj], date_set.dates.order_by('time'))
        self.assertItemsEqual([empty_obj, obj], date_set.dates.order_by('-time'))

    def test_update_with_f_expr(self):
        i = IntegerModel.objects.create(integer_field=1000)
        qs = IntegerModel.objects.all()
        qs.update(integer_field=models.F('integer_field') + 1)

        self.assertRaises(IntegerModel.DoesNotExist, IntegerModel.objects.get, integer_field=1000)
        i = IntegerModel.objects.get(pk=i.pk)
        self.assertEqual(1001, i.integer_field)

    def test_save_with_f_expr(self):
        i = IntegerModel.objects.create(integer_field=1000)

        i.integer_field = models.F('integer_field') + 1
        i.save()

        self.assertRaises(IntegerModel.DoesNotExist, IntegerModel.objects.get, integer_field=1000)
        i = IntegerModel.objects.get(pk=i.pk)
        self.assertEqual(1001, i.integer_field)

    def test_update_atomic(self):
        """
        Test concurrent updates are atomic.
        """
        properties_to_set = {
            "field1": "f1_val",
            "field2": "f2_val",
            "field3": "f3_val",
            "field4": "f4_val",
            "field5": "f5_val",
            "field6": "f6_val",
            "field7": "f7_val",
            "field8": "f8_val",
            "field9": "f9_val",
            "field10": "f10_val",
        }
        thread_property_map = {}
        original_fruit = ManyFields.objects.create()

        with concurrent.futures.ThreadPoolExecutor(max_workers=len(properties_to_set)) as executor:
            for property, value in properties_to_set.items():
                thread_property_map[property] = executor.submit(
                    set_test_many_fields_model_field, original_fruit, property, value)
        concurrent.futures.wait(thread_property_map.values())

        original_fruit.refresh_from_db()

        futures_exceptions = [future.exception() for future in thread_property_map.values()]
        number_successful_writes = len(list(itertools.filterfalse(None, futures_exceptions)))
        self.assertGreater(number_successful_writes, 0, 'It should save at least once')

        for property, future in thread_property_map.items():
            exception = future.exception()
            if exception:
                self.assertIsInstance(exception, DatabaseError)
            else:
                self.assertEqual(getattr(original_fruit, property), properties_to_set.get(property))

    def test_is_null_filtering(self):
        o1 = ModelWithNullableCharField.objects.create(some_id=1, field1="test")
        o2 = ModelWithNullableCharField.objects.create(some_id=1)
        o3 = ModelWithNullableCharField.objects.create(some_id=1)

        self.assertEqual(
            {o2, o3},
            set(ModelWithNullableCharField.objects.filter(field1__isnull=True))
        )

        self.assertEqual(
            {o1},
            set(ModelWithNullableCharField.objects.filter(field1__isnull=False))
        )

    @datastore_only
    def test_ordering_by_scatter_property(self):
        try:
            list(TestFruit.objects.order_by(Scatter()))
        except:  # noqa
            raise
            logging.exception("Error sorting on __scatter__")
            self.fail("Unable to sort on __scatter__ property like we should")

    def test_text_fields_arent_indexed(self):
        # String that is too long for Datastore or Firestore indexing
        long_string = "".join([random.choice(letters) for x in range(1600)])
        self.assertTrue(len(long_string) > 1500)
        fruit = TestFruit.objects.create(pk="test", text_field=long_string)
        fruit.refresh_from_db()
        fruit.save()

    def test_bytes_fields_arent_indexed(self):
        # String that is too long for Datastore or Firestore indexing
        long_string = "".join([random.choice(letters) for x in range(1600)])
        long_string = long_string.encode("ascii")
        fruit = TestFruit.objects.create(pk="test", binary_field=long_string)
        fruit.refresh_from_db()
        fruit.save()

    def test_ordering_on_non_indexed_fields_not_supported(self):
        self.assertRaises(NotSupportedError, list, TestFruit.objects.order_by("text_field"))
        self.assertRaises(NotSupportedError, list, TestFruit.objects.order_by("binary_field"))

    def test_ordering_on_sparse_field(self):
        """
        Case when sorting on field that is not present in all of
        Datastore entities. That can easily happen when you added
        new field to model and did not populated all existing entities
        """
        rpc = default_connection.connection

        # Clean state
        self.assertEqual(TestFruit.objects.count(), 0)

        # Put constistent instances to Datastore
        TestFruit.objects.create(name='a', color='a')
        TestFruit.objects.create(name='b', color='b')

        # Put inconsistent instances to Datastore
        # Color fields is missing (not even None)
        # we need more than 1 so we explore all sorting branches
        values = {'name': 'c'}
        entity = Entity(Key(TestFruit._meta.db_table, 'c', namespace=default_connection.namespace))
        entity._properties.update(values)
        rpc.put(entity)

        values = {'name': 'd'}
        entity = Entity(Key(TestFruit._meta.db_table, 'd', namespace=default_connection.namespace))
        entity._properties.update(values)
        rpc.put(entity)

        # Ok, we can get all 4 instances
        self.assertEqual(TestFruit.objects.count(), 4)

        all_names = ['a', 'b', 'c', 'd']
        fruits = list(
            TestFruit.objects.filter(name__in=all_names).order_by('color', 'name')
        )
        values = [(x.color or "", x.name) for x in fruits]
        self.assertEqual(sorted(values), values)

        # Test the ordering of the results.  The ones with a color of None should come back first,
        # and of the ones with color=None, they should be ordered by name
        # Missing one (None) as first
        expected_fruits = [
            ('c', None), ('d', None), ('a', 'a'), ('b', 'b'),
        ]

        self.assertEqual(
            [(fruit.name, fruit.color) for fruit in fruits],
            expected_fruits,
        )

    def test_double_order_on_same_key(self):

        a = TestFruit.objects.create(name='a', color='a')
        b = TestFruit.objects.create(name='b', color='b')

        self.assertEqual(list(TestFruit.objects.order_by('color', 'color')), [a, b])

    def test_order_on_field_after_key(self):

        a = TestFruit.objects.create(name='a', color='a')
        b = TestFruit.objects.create(name='b', color='b')

        self.assertEqual(list(TestFruit.objects.order_by('name', 'color')), [a, b])

    def test_duration_field_stored_as_float(self):
        """ See issue #512.  We have a bug report that the DurationField comes back as None when
            the value is set to a particular value which is roughly 3 days. This is caused by it
            being stored as a float instead of an int in the DB.
        """
        td2 = datetime.timedelta(days=2)
        # If the duration value is stored as a float instead of an int then this particular duration
        # will cause django.db.backends.base.operations.BaseDatabaseOperations.convert_durationfield_value
        # to return the value as None
        td3 = datetime.timedelta(days=3, seconds=14658, microseconds=886540)
        durations_as_2 = DurationModel.objects.create(
            duration_field=td2,
            duration_field_nullable=td2
        )
        durations_as_3 = DurationModel.objects.create(
            duration_field=td3,
            duration_field_nullable=td3
        )
        self.assertEqual(durations_as_2.duration_field, td2)
        self.assertEqual(durations_as_2.duration_field_nullable, td2)
        self.assertEqual(durations_as_3.duration_field, td3)
        self.assertEqual(durations_as_3.duration_field_nullable, td3)
        durations_as_2 = DurationModel.objects.get(pk=durations_as_2.pk)
        durations_as_3 = DurationModel.objects.get(pk=durations_as_3.pk)
        self.assertEqual(durations_as_2.duration_field, td2)
        self.assertEqual(durations_as_2.duration_field_nullable, td2)
        self.assertEqual(durations_as_3.duration_field, td3)
        self.assertEqual(durations_as_3.duration_field_nullable, td3)
        # And just for good measure, check the raw value in the datastore
        key = default_connection.connection.new_key(
            DurationModel._meta.db_table, durations_as_3.pk, namespace=default_connection.namespace
        )
        entity = default_connection.connection.get(key)
        self.assertTrue(isinstance(entity['duration_field'], int))

    def test_datetime_and_time_fields_precision_for_projection_queries(self):
        """
        The returned datetime and time values should include microseconds.
        See issue #707.
        """

        t = datetime.time(22, 13, 50, 541022)
        dt = make_aware(datetime.datetime(2016, 5, 27, 18, 40, 12, 927371))
        NullDate.objects.create(time=t, datetime=dt)
        result = list(NullDate.objects.all().values_list('time', 'datetime'))
        expected = [(t, dt)]

        self.assertItemsEqual(result, expected)

    @override_settings(USE_TZ=False)
    def test_datetime_and_time_fields_precision_for_projection_queries_no_tz(self):
        """
        The returned datetime and time values should include microseconds.
        See issue #707.
        """

        t = datetime.time(22, 13, 50, 541022)
        dt = datetime.datetime(2016, 5, 27, 18, 40, 12, 927371)
        NullDate.objects.create(time=t, datetime=dt)
        result = list(NullDate.objects.all().values_list('time', 'datetime'))
        expected = [(t, dt)]

        self.assertItemsEqual(result, expected)

    @override_settings(TIMEZONE="Europe/Rome")
    def test_datetime_and_time_fields_precision_for_projection_queries_other_tz(self):
        """
        The returned datetime and time values should include microseconds.
        See issue #707.
        """

        t = datetime.time(22, 13, 50, 541022)
        dt = make_aware(datetime.datetime(2016, 5, 27, 18, 40, 12, 927371))
        NullDate.objects.create(time=t, datetime=dt)
        result = list(NullDate.objects.all().values_list('time', 'datetime'))
        expected = [(t, dt)]

        self.assertItemsEqual(result, expected)

    def test_filter_with_empty_q(self):
        t1 = TestUser.objects.create(username='foo', field2='bar')
        condition = Q() | Q(username='foo')
        self.assertEqual(t1, TestUser.objects.filter(condition).first())

        condition = Q()
        self.assertEqual(t1, TestUser.objects.filter(condition).first())

    def test_only_defer_does_projection(self):
        with sleuth.watch("gcloudc.db.backends.common.base.query.Query.__init__") as query:
            list(TestUser.objects.only("username").all())
            self.assertFalse(query.calls[0].kwargs.get("only_return_keys"))
            self.assertItemsEqual(query.calls[0].kwargs.get("only_return_fields", []), ["username"])

        with sleuth.watch("gcloudc.db.backends.common.base.query.Query.__init__") as query:
            list(TestUser.objects.only("pk").all())
            self.assertFalse(query.calls[0].kwargs.get("only_return_fields"))
            self.assertTrue(query.calls[0].kwargs.get("only_return_keys"))

        with sleuth.watch("gcloudc.db.backends.common.base.query.Query.__init__") as query:
            list(TestUser.objects.values("pk"))
            self.assertFalse(query.calls[0].kwargs.get("only_return_fields"))
            self.assertTrue(query.calls[0].kwargs.get("only_return_keys"))

        with sleuth.watch("gcloudc.db.backends.common.base.query.Query.__init__") as query:
            list(TestUser.objects.defer("username").all())
            self.assertFalse(query.calls[0].kwargs.get("only_return_keys"))
            self.assertTrue(query.calls[0].kwargs["only_return_fields"])
            self.assertFalse("username" in query.calls[0].kwargs["only_return_fields"])

    def test_chaining_none_filter(self):
        t1 = TestUser.objects.create()
        self.assertFalse(TestUser.objects.none().filter(pk=t1.pk))

    def test_values_list_returns_str_strings(self):
        TestUser.objects.create(username=u"Łukasz")
        self.assertEqual(str, type(TestUser.objects.get().username))
        self.assertEqual(str, type(TestUser.objects.values_list("username", flat=True)[0]))


class ModelFormsetTest(TestCase):
    def test_reproduce_index_error(self):
        class TestModelForm(ModelForm):
            class Meta:
                model = TestUser
                fields = ("username", "email", "field2")

        test_model = TestUser.objects.create(username='foo', field2='bar')
        TestModelFormSet = modelformset_factory(TestUser, form=TestModelForm, extra=0)
        TestModelFormSet(queryset=TestUser.objects.filter(pk=test_model.pk))

        data = {
            'form-INITIAL_FORMS': 0,
            'form-MAX_NUM_FORMS': 0,
            'form-TOTAL_FORMS': 0,
            'form-0-id': test_model.id,
            'form-0-field1': 'foo_1',
            'form-0-field2': 'bar_1',
        }
        factory = RequestFactory()
        request = factory.post('/', data=data)

        TestModelFormSet(request.POST, request.FILES)


urlpatterns = [
    path("/", lambda x: None)
]


class CacheTests(TestCase):

    def test_cache_set(self):
        cache.set('test?', 'yes!')
        self.assertEqual(cache.get('test?'), 'yes!')

    def test_cache_timeout(self):
        cache.set('test?', 'yes!', 1)
        import time
        time.sleep(1)
        self.assertEqual(cache.get('test?'), None)


def compare_markers(list1, list2):
    return (
        sorted([(x.key(), x.instance) for x in list1]) == sorted([(x.key(), x.instance) for x in list2])
    )


class ConstraintTests(TestCase):
    """
        Tests for unique constraint handling
    """

    def test_conflicting_insert_throws_integrity_error(self):
        ModelWithUniques.objects.create(name="One")

        with self.assertRaises(IntegrityError):
            ModelWithUniques.objects.create(name="One")

        # An insert with a specified ID enters a different code path
        # so we need to ensure it works
        ModelWithUniques.objects.create(id=555, name="Two")

        with self.assertRaises(IntegrityError):
            ModelWithUniques.objects.create(name="Two")

        # Make sure that bulk create works properly
        ModelWithUniques.objects.bulk_create([
            ModelWithUniques(name="Three"),
            ModelWithUniques(name="Four"),
            ModelWithUniques(name="Five"),
        ])

        with self.assertRaises(IntegrityError):
            ModelWithUniques.objects.create(name="Four")

    def test_integrity_error_message_correct(self):
        """ Check that the IntegrityError messages mentions the correct field(s). """

        # Create a conflict on `unique_field`
        obj1 = UniqueModel.objects.create(unique_field="One")
        try:
            UniqueModel.objects.create(unique_field="One", unique_combo_one=1)
        except IntegrityError as e:
            self.assertTrue("unique_field" in str(e))

        # Create a conflict on `unique_relation`
        UniqueModel.objects.create(unique_relation=obj1, unique_field="two", unique_combo_one=2)
        try:
            UniqueModel.objects.create(unique_relation=obj1, unique_field="three", unique_combo_one=3)
        except IntegrityError as e:
            self.assertTrue("unique_relation" in str(e))

        # Create a conflict on a unique_together combo`
        UniqueModel.objects.create(unique_field="four", unique_combo_one=4, unique_combo_two="five")
        try:
            UniqueModel.objects.create(unique_field="five", unique_combo_one=4, unique_combo_two="five")
        except IntegrityError as e:
            self.assertTrue("unique_combo_one" in str(e))
            self.assertTrue("unique_combo_two" in str(e))

    def test_table_flush_clears_markers_for_that_table(self):
        ModelWithUniques.objects.create(name="One")
        UniqueModel.objects.create(unique_field="One")

        FlushCommand(None, ModelWithUniques._meta.db_table, default_connection).execute()

        ModelWithUniques.objects.create(name="One")

        with self.assertRaises(IntegrityError):
            UniqueModel.objects.create(unique_field="One")

    def test_recently_deleted_unique_doesnt_come_back(self):
        instance = ModelWithUniques.objects.create(name="One")
        instance.delete()
        self.assertEqual(0, ModelWithUniques.objects.filter(name="One").count())
        self.assertFalse(ModelWithUniques.objects.filter(name="One").exists())
        self.assertFalse(list(ModelWithUniques.objects.all()))  # Triple-check

    def test_conflicting_update_throws_integrity_error(self):
        ModelWithUniques.objects.create(name="One")

        instance = ModelWithUniques.objects.create(name="Two")
        with self.assertRaises(IntegrityError):
            instance.name = "One"
            instance.save()

    def test_unique_combinations_are_returned_correctly(self):
        combos_one = _unique_combinations(ModelWithUniquesOnForeignKey, ignore_pk=True)
        combos_two = _unique_combinations(ModelWithUniquesOnForeignKey, ignore_pk=False)

        self.assertEqual([['name', 'related_name'], ['name'], ['related_name']], combos_one)
        self.assertEqual([['name', 'related_name'], ['id'], ['name'], ['related_name']], combos_two)

        e1 = Entity(Key(ModelWithUniquesOnForeignKey._meta.db_table, 1))
        e1["name"] = "One"
        e1["related_name_id"] = 1

        ids_one = unique_identifiers_from_entity(ModelWithUniquesOnForeignKey, e1, ignore_pk=False)

        self.assertItemsEqual([
            u'tests_modelwithuniquesonforeignkey|id:1',
            u'tests_modelwithuniquesonforeignkey|name:06c2cea18679d64399783748fa367bdd',
            u'tests_modelwithuniquesonforeignkey|related_name_id:1',
            u'tests_modelwithuniquesonforeignkey|name:06c2cea18679d64399783748fa367bdd|related_name_id:1'
        ], ids_one)

    def test_list_field_unique_constaints(self):
        instance1 = UniqueModel.objects.create(unique_field=1, unique_combo_one=1, unique_list_field=["A", "C"])

        with self.assertRaises((IntegrityError, DataError)):
            UniqueModel.objects.create(unique_field=2, unique_combo_one=2, unique_list_field=["A"])

        instance2 = UniqueModel.objects.create(unique_field=2, unique_combo_one=2, unique_list_field=["B"])

        instance2.unique_list_field = instance1.unique_list_field

        with self.assertRaises((IntegrityError, DataError)):
            instance2.save()

        instance1.unique_list_field = []
        instance1.save()

        instance2.save()

    def test_list_field_unique_constraints_validation(self):
        instance1 = UniqueModel(
            unique_set_field={"A"},
            unique_together_list_field=[1],
            unique_field=1,
            unique_combo_one=1,
            unique_list_field=["A", "C"]
        )

        # Without a custom mixin, Django can't construct a unique validation query for a list field
        self.assertRaises(ValueError, instance1.full_clean)
        UniqueModel.__bases__ = (UniquenessMixin,) + UniqueModel.__bases__
        instance1.full_clean()
        instance1.save()

        # Check the uniqueness mixing works with long lists
        instance1.unique_list_field = [x for x in range(31)]
        try:
            instance1.full_clean()
        except NotSupportedError:
            self.fail("Couldn't run unique check on long list field")
            return

        instance2 = UniqueModel(
            unique_set_field={"B"},
            unique_together_list_field=[2],
            unique_field=2,
            unique_combo_one=2,
            unique_list_field=["B", "C"]  # duplicate value C!
        )

        self.assertRaises(ValidationError, instance2.full_clean)
        UniqueModel.__bases__ = (models.Model,)

    def test_set_field_unique_constraints(self):
        instance1 = UniqueModel.objects.create(unique_field=1, unique_combo_one=1, unique_set_field={"A", "C"})

        with self.assertRaises((IntegrityError, DataError)):
            UniqueModel.objects.create(unique_field=2, unique_combo_one=2, unique_set_field={"A"})

        instance2 = UniqueModel.objects.create(unique_field=2, unique_combo_one=2, unique_set_field={"B"})

        instance2.unique_set_field = instance1.unique_set_field

        with self.assertRaises((IntegrityError, DataError)):
            instance2.save()

        instance1.unique_set_field = set()
        instance1.save()

        instance2.save()

        instance2.unique_set_field = set()
        instance2.save()  # You can have two fields with empty sets

    def test_unique_constraints_on_model_with_long_str_pk(self):
        """ Check that an object with a string-based PK of 500 characters (the max that GAE allows)
            can still have unique constraints pointing at it.  (See #242.)
        """
        obj = UniqueModelWithLongPK(pk="x" * 500, unique_field=1)
        obj.save()
        duplicate = UniqueModelWithLongPK(pk="y" * 500, unique_field=1)
        self.assertRaises(IntegrityError, duplicate.save)


class EdgeCaseTests(TestCase):
    def setUp(self):
        super(EdgeCaseTests, self).setUp()

        add_special_index(default_connection, TestUser, "username", IExactIndexer(), "iexact")

        self.u1 = TestUser.objects.create(
            username="A",
            email="test@example.com",
            last_login=datetime.datetime.now().date(),
            id=1, first_name="A", second_name="A"
        )

        self.u2 = TestUser.objects.create(
            username="B",
            email="test@example.com",
            last_login=datetime.datetime.now().date(),
            id=2, first_name="B", second_name="B"
        )

        self.u3 = TestUser.objects.create(
            username="C",
            email="test2@example.com",
            last_login=datetime.datetime.now().date(),
            id=3, first_name="C", second_name="C"
        )

        self.u4 = TestUser.objects.create(
            username="D",
            email="test3@example.com",
            last_login=datetime.datetime.now().date(),
            id=4, first_name="D", second_name="D"
        )

        self.u5 = TestUser.objects.create(
            username="E",
            email="test3@example.com",
            last_login=datetime.datetime.now().date(),
            id=5, first_name="E", second_name="E"
        )

        self.apple = TestFruit.objects.create(name="apple", color="red")
        self.banana = TestFruit.objects.create(name="banana", color="yellow")

    def test_querying_by_date(self):
        instance1 = ModelWithDates.objects.create(start=datetime.date(2014, 1, 1), end=datetime.date(2014, 1, 20))
        instance2 = ModelWithDates.objects.create(start=datetime.date(2014, 2, 1), end=datetime.date(2014, 2, 20))

        self.assertEqual(instance1, ModelWithDates.objects.get(start__lt=datetime.date(2014, 1, 2)))
        self.assertEqual(2, ModelWithDates.objects.filter(start__lt=datetime.date(2015, 1, 1)).count())

        self.assertEqual(instance2, ModelWithDates.objects.get(start__gt=datetime.date(2014, 1, 2)))
        self.assertEqual(instance2, ModelWithDates.objects.get(start__gte=datetime.date(2014, 2, 1)))

    def projection_plus_keys_filtering(self):
        """
            If you do a query like this:

            MyModel.objects.filter(pk__in=[1, 2]).filter(field1="Bananas").values_list("id", "someotherfield")

            Then a projection query is run. The problem is that the entities returned only include
            "id" and "someotherfield"
            but not "field1". Our entity-matches-query code should not run in this situation as we pass
            all filters to the ancestor queries and so any entities returned should match.
        """

        user = TestUser.objects.create(username="test", email="test@example.com")

        self.assertItemsEqual(
            [(user.pk, user.username)],
            TestUser.objects.filter(
                pk__in=[user.pk, user.pk + 1]
            ).filter(
                email="test@example.com"
            ).values_list("id", "username")
        )

    def test_double_starts_with(self):
        qs = TestUser.objects.filter(
            username__startswith='Hello'
        ) | TestUser.objects.filter(username__startswith='Goodbye')

        self.assertEqual(0, qs.count())

        TestUser.objects.create(username="Hello", first_name="A")
        self.assertEqual(1, qs.count())

        TestUser.objects.create(username="Goodbye", first_name="B")
        self.assertEqual(2, qs.count())

        TestUser.objects.create(username="Hello and Goodbye", first_name="C")
        self.assertEqual(3, qs.count())

    def test_impossible_starts_with(self):
        TestUser.objects.create(username="Hello", first_name="A")
        TestUser.objects.create(username="Goodbye", first_name="B")
        TestUser.objects.create(username="Hello and Goodbye", first_name="C")

        qs = TestUser.objects.filter(
            username__startswith='Hello'
        ) & TestUser.objects.filter(username__startswith='Goodbye')
        self.assertEqual(0, qs.count())

    def test_datetime_contains(self):
        """
            Django allows for __contains on datetime field, so that you can search for a specific
            date. This is probably just because SQL allows querying it on a string, and contains just
            turns into a like query. This test just makes sure we behave the same
        """

        fake_future_datetime = datetime.datetime(2100, 2, 10, 11, 59, 59, tzinfo=tz.utc)
        fake_future_date = datetime.date(2100, 2, 10)
        today_datetime = datetime.datetime.now()
        today_date = datetime.date.today()

        DateTimeModel.objects.create(datetime_field=fake_future_datetime, date_field=fake_future_date)

        instance_today_1 = DateTimeModel.objects.create(
            datetime_field=today_datetime,
            date_field=today_date
        )

        by_datetime = list(DateTimeModel.objects.filter(
            datetime_field__contains=today_date
        ))

        # Make sure that if we query a datetime on a date it is properly returned
        self.assertItemsEqual(
            [instance_today_1],
            by_datetime
        )

        by_year = list(DateTimeModel.objects.filter(date_field__contains=today_date.year))
        self.assertItemsEqual([instance_today_1], by_year)

        # Try with 2 mathing entities
        instance_today_2 = DateTimeModel.objects.create(
            datetime_field=today_datetime,
            date_field=today_date
        )

        by_datetime = list(DateTimeModel.objects.filter(
            datetime_field__contains=today_date
        ))

        # Make sure that if we query a datetime on a date it is properly returned
        self.assertItemsEqual(
            [instance_today_1, instance_today_2],
            by_datetime
        )

        by_year = list(DateTimeModel.objects.filter(date_field__contains=today_date.year))
        self.assertItemsEqual([instance_today_1, instance_today_2], by_year)

    # See https://gitlab.com/potato-oss/google-cloud/django-gcloud-connectors/issues/2
    def test_datetime_compare(self):
        from django.utils import timezone
        fake_future = datetime.datetime(2100, 2, 10, 11, 59, 59, tzinfo=tz.utc)
        fake_now = timezone.datetime(2020, 2, 10, 11, 59, 59)

        instance = DateTimeModel.objects.create(
            datetime_field=fake_future,
            date_field=datetime.date.today(),
        )

        fetched = DateTimeModel.objects.get(datetime_field__gt=fake_now, pk=instance.pk)

        self.assertEqual(fetched, instance)

    def test_combinations_of_special_indexes(self):
        qs = TestUser.objects.filter(username__iexact='Hello') | TestUser.objects.filter(username__contains='ood')

        self.assertEqual(0, qs.count())

        TestUser.objects.create(username="Hello", first_name="A")
        self.assertEqual(1, qs.count())

        TestUser.objects.create(username="Goodbye", first_name="B")
        self.assertEqual(2, qs.count())

        TestUser.objects.create(username="Hello and Goodbye", first_name="C")
        self.assertEqual(3, qs.count())

    def test_multi_table_inheritance(self):

        parent = MultiTableParent.objects.create(parent_field="parent1")
        child1 = MultiTableChildOne.objects.create(parent_field="child1", child_one_field="child1")
        child2 = MultiTableChildTwo.objects.create(parent_field="child2", child_two_field="child2")

        self.assertEqual(3, MultiTableParent.objects.count())
        self.assertItemsEqual(
            [parent.pk, child1.pk, child2.pk],
            list(MultiTableParent.objects.values_list('pk', flat=True))
        )
        self.assertEqual(1, MultiTableChildOne.objects.count())
        self.assertEqual(child1, MultiTableChildOne.objects.get())

        self.assertEqual(1, MultiTableChildTwo.objects.count())
        self.assertEqual(child2, MultiTableChildTwo.objects.get())

        self.assertEqual(child2, MultiTableChildTwo.objects.get(pk=child2.pk))
        self.assertTrue(MultiTableParent.objects.filter(pk=child2.pk).exists())

    def test_anding_pks(self):
        results = TestUser.objects.filter(id__exact=self.u1.pk).filter(id__exact=self.u2.pk)
        self.assertEqual(list(results), [])

    def test_filter_by_key_then_projection(self):
        results = list(TestFruit.objects.filter(name="apple").values_list("pk", "color"))
        self.assertEqual(len(results), 1)
        self.assertEqual(results[0], ("apple", "red"))

    def test_unusual_queries(self):

        results = list(TestFruit.objects.filter(name__in=["apple", "orange"]))
        self.assertEqual(1, len(results))
        self.assertItemsEqual(["apple"], [x.name for x in results])

        results = TestFruit.objects.filter(name__in=["apple", "banana"])
        self.assertEqual(2, len(results))
        self.assertItemsEqual(["apple", "banana"], [x.name for x in results])

        results = TestFruit.objects.filter(name__in=["apple", "banana"]).values_list('pk', 'color')
        self.assertEqual(2, len(results))
        self.assertItemsEqual([(self.apple.pk, self.apple.color), (self.banana.pk, self.banana.color)], results)

        results = TestUser.objects.all()
        self.assertEqual(5, len(results))

        results = TestUser.objects.filter(username__in=["A", "B"])
        self.assertEqual(2, len(results))
        self.assertItemsEqual(["A", "B"], [x.username for x in results])

        results = TestUser.objects.filter(username__in=["A", "B"]).exclude(username="A")
        self.assertEqual(1, len(results), results)
        self.assertItemsEqual(["B"], [x.username for x in results])

        results = TestUser.objects.filter(username__lt="E")
        self.assertEqual(4, len(results))
        self.assertItemsEqual(["A", "B", "C", "D"], [x.username for x in results])

        results = TestUser.objects.filter(username__lte="E")
        self.assertEqual(5, len(results))

        # Double exclude on different properties not supported
        with self.assertRaises(NotSupportedError):
            # FIXME: This should raise a NotSupportedError, but at the moment it's thrown too late in
            # the process and so Django wraps it as a DataError
            list(TestUser.objects.exclude(username="E").exclude(email="A"))

        results = list(TestUser.objects.exclude(username="E").exclude(username="A"))
        self.assertItemsEqual(["B", "C", "D"], [x.username for x in results])

        results = TestUser.objects.filter(username="A", email="test@example.com")
        self.assertEqual(1, len(results))

        results = TestUser.objects.filter(username__in=["A", "B"]).filter(username__in=["A", "B"])
        self.assertEqual(2, len(results))
        self.assertItemsEqual(["A", "B"], [x.username for x in results])

        results = list(TestUser.objects.filter(username__in=["A", "B"]).filter(username__in=["A"]))
        self.assertEqual(1, len(results))
        self.assertItemsEqual(["A"], [x.username for x in results])

        results = TestUser.objects.filter(pk__in=[self.u1.pk, self.u2.pk]).filter(username__in=["A"])
        self.assertEqual(1, len(results))
        self.assertItemsEqual(["A"], [x.username for x in results])

        results = TestUser.objects.filter(username__in=["A"]).filter(pk__in=[self.u1.pk, self.u2.pk])
        self.assertEqual(1, len(results))
        self.assertItemsEqual(["A"], [x.username for x in results])

        results = list(TestUser.objects.all().exclude(username__in=["A"]))
        self.assertItemsEqual(["B", "C", "D", "E"], [x.username for x in results])

        results = list(TestFruit.objects.filter(name='apple', color__in=[]))
        self.assertItemsEqual([], results)

        results = list(TestUser.objects.all().exclude(username__in=[]))
        self.assertEqual(5, len(results))
        self.assertItemsEqual(["A", "B", "C", "D", "E"], [x.username for x in results])

        results = list(TestUser.objects.all().exclude(username__in=[]).filter(username__in=["A", "B"]))
        self.assertEqual(2, len(results))
        self.assertItemsEqual(["A", "B"], [x.username for x in results])

        results = list(TestUser.objects.all().filter(username__in=["A", "B"]).exclude(username__in=[]))
        self.assertEqual(2, len(results))
        self.assertItemsEqual(["A", "B"], [x.username for x in results])

    def test_empty_string_key(self):
        # Creating
        with self.assertRaises(IntegrityError):
            TestFruit.objects.create(name='')

        # Getting
        with self.assertRaises(TestFruit.DoesNotExist):
            TestFruit.objects.get(name='')

        # Filtering
        results = list(TestFruit.objects.filter(name='').order_by("name"))
        self.assertItemsEqual([], results)

        # Combined filtering
        results = list(TestFruit.objects.filter(name='', color='red').order_by("name"))
        self.assertItemsEqual([], results)

        # IN query
        results = list(TestFruit.objects.filter(name__in=['', 'apple']))
        self.assertItemsEqual([self.apple], results)

    def test_sorting_by_filtered_field(self):
        user = TestUser.objects.create(
            username="F",
            email="test4@example.com",
            last_login=(datetime.datetime.now() - datetime.timedelta(days=1)).date(),
            id=6, first_name="F", second_name="F"
        )

        yesterday = (datetime.datetime.now() - datetime.timedelta(days=1)).date()

        # Filtering
        results = list(TestUser.objects.filter(last_login=yesterday).order_by("-last_login"))
        self.assertItemsEqual([user], results)

    def test_or_queryset(self):
        """
            This constructs an OR query, this is currently broken in the parse_where_and_check_projection
            function. WE MUST FIX THIS!
        """
        q1 = TestUser.objects.filter(username="A")
        q2 = TestUser.objects.filter(username="B")

        self.assertItemsEqual([self.u1, self.u2], list(q1 | q2))

    def test_or_q_objects(self):
        """ Test use of Q objects in filters. """
        query = TestUser.objects.filter(Q(username="A") | Q(username="B"))
        self.assertItemsEqual([self.u1, self.u2], list(query))

    def test_extra_select(self):
        results = TestUser.objects.filter(username='A').extra(select={'is_a': "username = 'A'"})
        self.assertEqual(1, len(results))
        self.assertItemsEqual([True], [x.is_a for x in results])

        results = TestUser.objects.all().exclude(username='A').extra(select={'is_a': "username = 'A'"})
        self.assertEqual(4, len(results))
        self.assertEqual(not any([x.is_a for x in results]), True)

        # Up for debate
        # results = User.objects.all().extra(select={'truthy': 'TRUE'})
        # self.assertEqual(all([x.truthy for x in results]), True)

        results = TestUser.objects.all().extra(select={'truthy': True})
        self.assertEqual(all([x.truthy for x in results]), True)

    def test_counts(self):
        self.assertEqual(5, TestUser.objects.count())
        self.assertEqual(2, TestUser.objects.filter(email="test3@example.com").count())
        self.assertEqual(3, TestUser.objects.exclude(email="test3@example.com").count())
        self.assertEqual(1, TestUser.objects.filter(username="A").exclude(email="test3@example.com").count())
        self.assertEqual(3, TestUser.objects.exclude(username="E").exclude(username="A").count())

        self.assertEqual(3, TestUser.objects.exclude(username__in=["A", "B"]).count())
        self.assertEqual(0, TestUser.objects.filter(email="test@example.com").exclude(username__in=["A", "B"]).count())

    def test_exclude_with__in(self):
        self.assertEqual(
            set([self.u3, self.u4, self.u5]),
            set(list(TestUser.objects.exclude(username__in=["A", "B"])))
        )

    def test_deletion(self):
        count = TestUser.objects.count()
        self.assertTrue(count)

        TestUser.objects.filter(username="A").delete()
        self.assertEqual(count - 1, TestUser.objects.count())

        TestUser.objects.filter(username="B").exclude(username="B").delete()  # Should do nothing
        self.assertEqual(count - 1, TestUser.objects.count())

        TestUser.objects.all().delete()
        count = TestUser.objects.count()
        self.assertFalse(count)

    def test_double_delete(self):
        u1 = TestUser.objects.get(username="A")
        u2 = TestUser.objects.get(username="A")

        u1.delete()
        u2.delete()

    def test_insert_with_existing_key(self):
        user = TestUser.objects.create(id=999, username="test1", last_login=datetime.datetime.now().date())
        self.assertEqual(999, user.pk)

        with self.assertRaises(IntegrityError):
            TestUser.objects.create(id=999, username="test2", last_login=datetime.datetime.now().date())

    def test_included_pks(self):
        ids = [TestUser.objects.get(username="B").pk, TestUser.objects.get(username="A").pk]
        results = TestUser.objects.filter(pk__in=ids).order_by("username")

        self.assertEqual(results[0], self.u1)
        self.assertEqual(results[1], self.u2)

    def test_select_related(self):
        """ select_related should be a no-op... for now """
        user = TestUser.objects.get(username="A")
        Permission.objects.create(user=user, perm="test_perm")
        select_related = [(p.perm, p.user.username) for p in user.permission_set.select_related()]
        self.assertEqual(user.username, select_related[0][1])

    def test_cross_selects(self):
        user = TestUser.objects.get(username="A")
        Permission.objects.create(user=user, perm="test_perm")
        with self.assertRaises(NotSupportedError):
            perms = list(Permission.objects.all().values_list("user__username", "perm"))
            self.assertEqual("A", perms[0][0])

    def test_invalid_id_value_on_insert(self):
        user = TestUser.objects.get(username="A")
        # pass in a User object as if it's an int
        permission = Permission(user_id=user, perm="test_perm")
        with self.assertRaises(TypeError):
            permission.save(force_insert=True)

    @datastore_only
    def test_values_list_on_pk_does_keys_only_query_ds(self):
        with sleuth.watch("google.cloud.datastore.query.Query.keys_only") as keys_only:
            list(TestUser.objects.all().values_list('pk', flat=True))
            self.assertTrue(keys_only.called)
            self.assertEqual(5, len(TestUser.objects.all().values_list('pk')))

    @firestore_only
    def test_values_list_on_pk_does_keys_only_query_fs(self):
        with sleuth.watch("google.cloud.firestore.Query.select") as keys_only:
            list(TestUser.objects.all().values_list('pk', flat=True))
            self.assertTrue(keys_only.called)
            self.assertEqual(5, len(TestUser.objects.all().values_list('pk')))

    def test_iexact(self):
        user = TestUser.objects.get(username__iexact="a")
        self.assertEqual("A", user.username)

        add_special_index(default_connection, IntegerModel, "integer_field", IExactIndexer(), "iexact")
        IntegerModel.objects.create(integer_field=1000)
        integer_model = IntegerModel.objects.get(integer_field__iexact=str(1000))
        self.assertEqual(integer_model.integer_field, 1000)

        user = TestUser.objects.get(id__iexact=str(self.u1.id))
        self.assertEqual("A", user.username)

    def test_iexact_containing_underscores(self):
        add_special_index(default_connection, TestUser, "username", IExactIndexer(), "iexact")
        user = TestUser.objects.create(username="A_B", email="test@example.com")
        results = TestUser.objects.filter(username__iexact=user.username.lower())
        self.assertEqual(list(results), [user])

    def test_year(self):
        user = TestUser.objects.create(username="Z", email="z@example.com")
        user.last_login = datetime.datetime(2000, 1, 1, 0, 0, 0)
        user.save()

        self.assertEqual(len(TestUser.objects.filter(last_login__year=3000)), 0)
        self.assertEqual(TestUser.objects.filter(last_login__year=2000).first().pk, user.pk)

    def test_ordering(self):
        users = TestUser.objects.all().order_by("username")

        self.assertEqual(["A", "B", "C", "D", "E"], [x.username for x in users])

        users = TestUser.objects.all().order_by("-username")

        self.assertEqual(["A", "B", "C", "D", "E"][::-1], [x.username for x in users])

        with self.assertRaises(FieldError):
            users = list(TestUser.objects.order_by("bananas"))

        users = TestUser.objects.filter(id__in=[self.u2.id, self.u3.id, self.u4.id]).order_by('id')
        self.assertEqual(["B", "C", "D"], [x.username for x in users])

        users = TestUser.objects.filter(id__in=[self.u2.id, self.u3.id, self.u4.id]).order_by('-id')
        self.assertEqual(["D", "C", "B"], [x.username for x in users])

        users = TestUser.objects.filter(id__in=[self.u1.id, self.u5.id, self.u3.id]).order_by('id')
        self.assertEqual(["A", "C", "E"], [x.username for x in users])

        users = TestUser.objects.filter(id__in=[self.u4.id, self.u5.id, self.u3.id, self.u1.id]).order_by('-id')
        self.assertEqual(["E", "D", "C", "A"], [x.username for x in users])

    def test_dates_query(self):
        z_user = TestUser.objects.create(username="Z", email="z@example.com")
        z_user.last_login = datetime.date(2013, 4, 5)
        z_user.save()

        last_a_login = TestUser.objects.get(username="A").last_login

        dates = TestUser.objects.dates('last_login', 'year')

        self.assertItemsEqual(
            [datetime.date(2013, 1, 1), datetime.date(last_a_login.year, 1, 1)],
            dates
        )

        dates = TestUser.objects.dates('last_login', 'month')
        self.assertItemsEqual(
            [datetime.date(2013, 4, 1), datetime.date(last_a_login.year, last_a_login.month, 1)],
            dates
        )

        dates = TestUser.objects.dates('last_login', 'day')
        self.assertEqual(
            [datetime.date(2013, 4, 5), datetime.date(last_a_login.year, last_a_login.month, last_a_login.day)],
            list(dates)
        )

        dates = TestUser.objects.dates('last_login', 'day', order='DESC')
        self.assertEqual(
            [datetime.date(last_a_login.year, last_a_login.month, last_a_login.day), datetime.date(2013, 4, 5)],
            list(dates)
        )

    @override_settings(DJANGAE_MAX_QUERY_BRANCHES=30)
    def test_in_query(self):
        """ Test that the __in filter works, and that it cannot be used with more than 30 values,
            unless it's used on the PK field.
        """
        # Check that a basic __in query works
        results = list(TestUser.objects.filter(username__in=['A', 'B']))
        self.assertItemsEqual(results, [self.u1, self.u2])
        # Check that it also works on PKs
        results = list(TestUser.objects.filter(pk__in=[self.u1.pk, self.u2.pk]))
        self.assertItemsEqual(results, [self.u1, self.u2])
        # Check that using more than 30 items in an __in query not on the pk causes death
        query = TestUser.objects.filter(username__in=list([x for x in letters[:31]]))
        self.assertRaises(Exception, list, query)
        # Check that it's ok with PKs though
        query = TestUser.objects.filter(pk__in=list(range(1, 32)))
        list(query)
        # Check that it's ok joining filters with pks
        results = list(TestUser.objects.filter(
            pk__in=[self.u1.pk, self.u2.pk, self.u3.pk]).filter(pk__in=[self.u1.pk, self.u2.pk]))
        self.assertItemsEqual(results, [self.u1, self.u2])

    def test_self_relations(self):
        obj = SelfRelatedModel.objects.create()
        obj2 = SelfRelatedModel.objects.create(related=obj)
        self.assertEqual(list(obj.selfrelatedmodel_set.all()), [obj2])

    @skip("This test violates django 4.1+ rules")
    def test_self_relations_unsaved(self):
        """ Tests self-referencing relations (ForeignKey('self')) """

        obj = SelfRelatedModel()
        SelfRelatedModel(related=obj)
        self.assertEqual(list(obj.selfrelatedmodel_set.all()), [])

    def test_special_indexes_for_empty_fields(self):
        obj = TestFruit.objects.create(name='pear')
        indexes = ['icontains', 'contains', 'iexact', 'iendswith', 'endswith', 'istartswith', 'startswith']
        for index in indexes:
            add_special_index(
                default_connection,
                TestFruit,
                'color',
                get_indexer(TestFruit._meta.get_field("color"), index),
                index
            )
        obj.save()

    def test_special_indexes_for_unusually_long_values(self):
        obj = TestFruit.objects.create(name='pear', color='1234567890-=!@#$%^&*()_+qQWERwertyuiopasdfghjklzxcvbnm')
        indexes = ['icontains', 'contains', 'iexact', 'iendswith', 'endswith', 'istartswith', 'startswith']
        for index in indexes:
            add_special_index(
                default_connection,
                TestFruit,
                'color',
                get_indexer(TestFruit._meta.get_field("color"), index),
                index
            )
        obj.save()

        qry = TestFruit.objects.filter(color__contains='1234567890-=!@#$%^&*()_+qQWERwertyuiopasdfghjklzxcvbnm')
        self.assertEqual(len(list(qry)), 1)
        qry = TestFruit.objects.filter(color__contains='890-=!@#$')
        self.assertEqual(len(list(qry)), 1)
        qry = TestFruit.objects.filter(color__contains='1234567890-=!@#$%^&*()_+qQWERwertyui')
        self.assertEqual(len(list(qry)), 1)
        qry = TestFruit.objects.filter(color__contains='8901')
        self.assertEqual(len(list(qry)), 0)

        qry = TestFruit.objects.filter(color__icontains='1234567890-=!@#$%^&*()_+qQWERWERTYuiopasdfghjklzxcvbnm')
        self.assertEqual(len(list(qry)), 1)
        qry = TestFruit.objects.filter(color__icontains='890-=!@#$')
        self.assertEqual(len(list(qry)), 1)
        qry = TestFruit.objects.filter(color__icontains='1234567890-=!@#$%^&*()_+qQWERwertyuI')
        self.assertEqual(len(list(qry)), 1)
        qry = TestFruit.objects.filter(color__icontains='8901')
        self.assertEqual(len(list(qry)), 0)

    def test_values_list_on_distinct(self):
        TestFruit.objects.create(name="Kiwi", origin="New Zealand", color="Green")
        TestFruit.objects.create(name="Apple", origin="New Zealand", color="Green")
        TestFruit.objects.create(name="Grape", origin="New Zealand", color="Red")
        nz_colours = TestFruit.objects.filter(
            origin="New Zealand"
        ).distinct("color").values_list("color", flat=True)
        self.assertEqual(sorted(nz_colours), ["Green", "Red"])

    def test_values_list_on_distinct_with_limit(self):
        TestFruit.objects.create(name="Kiwi", origin="New Zealand", color="Green")
        TestFruit.objects.create(name="Apple", origin="New Zealand", color="Green")
        TestFruit.objects.create(name="Grape", origin="New Zealand", color="Red")

        nz_colours = TestFruit.objects.filter(
            origin="New Zealand"
        ).distinct("color").order_by('color').values_list("color", flat=True)[:1]

        self.assertEqual(sorted(nz_colours), ["Green"])

    def test_values_list_on_distinct_offset(self):
        TestFruit.objects.create(name="Kiwi", origin="New Zealand", color="Green")
        TestFruit.objects.create(name="Apple", origin="New Zealand", color="Green")
        TestFruit.objects.create(name="Grape", origin="New Zealand", color="Red")

        nz_colours = TestFruit.objects.filter(
            origin="New Zealand"
        ).distinct("color").order_by('color').values_list("color", flat=True)[1:]

        self.assertEqual(sorted(nz_colours), ["Red"])

    def test_values_list_on_distinct_limit_and_offset(self):
        TestFruit.objects.create(name="Kiwi", origin="New Zealand", color="Green")
        TestFruit.objects.create(name="Apple", origin="New Zealand", color="Green")
        TestFruit.objects.create(name="Orange", origin="New Zealand", color="Orange")
        TestFruit.objects.create(name="Orange2", origin="New Zealand", color="Orange")
        TestFruit.objects.create(name="Grape", origin="New Zealand", color="Red")
        TestFruit.objects.create(name="Apple2", origin="New Zealand", color="Red")
        TestFruit.objects.create(name="Banana", origin="New Zealand", color="Yellow")
        TestFruit.objects.create(name="Mango", origin="New Zealand", color="Yellow")

        nz_colours = TestFruit.objects.filter(
            origin="New Zealand"
        ).distinct("color").order_by('color').values_list("color", flat=True)[1:3]

        self.assertEqual(sorted(nz_colours), ["Orange", "Red"])

    def test_empty_key_lookups_work_correctly(self):
        t1 = TestFruit.objects.create(name="Kiwi", origin="New Zealand", color="Green")
        TestFruit.objects.create(name="Apple", origin="New Zealand", color="Green")

        self.assertEqual(
            t1,
            TestFruit.objects.exclude(name="Apple").exclude(name="").get(name="Kiwi")
        )

        self.assertFalse(TestFruit.objects.filter(name="", color="Green"))
        self.assertTrue(TestFruit.objects.filter(Q(name="") | Q(name="Kiwi")).filter(color="Green"))
        self.assertFalse(TestFruit.objects.filter(name="", color__gt="A"))
        self.assertEqual(4, TestFruit.objects.exclude(name="").count())

    def test_additional_indexes_respected(self):
        project, additional = indexing._project_special_indexes.copy(), indexing._app_special_indexes.copy()

        try:
            indexing._project_special_indexes = {}
            indexing._app_special_indexes = {
                TestFruit._meta.db_table: {"name": ["iexact"]}
            }

            t1 = TestFruit.objects.create(name="Kiwi", origin="New Zealand", color="Green")
            self.assertEqual(t1, TestFruit.objects.filter(name__iexact="kiwi").get())
            self.assertFalse(indexing._project_special_indexes)  # Nothing was added
        finally:
            indexing._project_special_indexes = project
            indexing._app_special_indexes = additional


class TestSpecialIndexers(TestCase):

    def setUp(self):
        super(TestSpecialIndexers, self).setUp()

        self.names = [
            'Ola', 'Adam', 'Luke', 'rob', 'Daniel', 'Ela', 'Olga', 'olek',
            'ola', 'Olaaa', 'OlaaA', 'Ola + Ola', '-Test-', '-test-'
        ]
        for name in self.names:
            SpecialIndexesModel.objects.create(name=name)

        self.lists = [
            self.names,
            ['Name', 'name', 'name + name'],
            ['-Tesst-'],
            ['-test-']
        ]
        for i, sample_list in enumerate(self.lists):
            SpecialIndexesModel.objects.create(name=i, sample_list=sample_list)

        self.qry = SpecialIndexesModel.objects.all()

    def test_iexact_lookup(self):
        for name in self.names:
            qry = self.qry.filter(name__iexact=name)
            self.assertEqual(len(qry), len([x for x in self.names if x.lower() == name.lower()]))

    def test_contains_lookup_and_icontains_lookup(self):
        tests = self.names + ['o', 'O', 'la']
        for name in tests:
            qry = self.qry.filter(name__contains=name)
            self.assertEqual(len(qry), len([x for x in self.names if name in x]))

            qry = self.qry.filter(name__icontains=name)
            self.assertEqual(len(qry), len([x for x in self.names if name.lower() in x.lower()]))

    def test_contains_lookup_on_charfield_subclass(self):
        """ Test that the __contains lookup also works on subclasses of the Django CharField, e.g.
            the custom Djangae CharField.
        """
        instance = SpecialIndexesModel.objects.create(name="whatever", nickname="Voldemort")
        query = SpecialIndexesModel.objects.filter(nickname__contains="demo")
        self.assertEqual(list(query), [instance])

    def test_endswith_lookup_and_iendswith_lookup(self):
        tests = self.names + ['a', 'A', 'aa']
        for name in tests:
            qry = self.qry.filter(name__endswith=name)
            self.assertEqual(len(qry), len([x for x in self.names if x.endswith(name)]))

            qry = self.qry.filter(name__iendswith=name)
            self.assertEqual(len(qry), len([x for x in self.names if x.lower().endswith(name.lower())]))

    def test_startswith_lookup_and_istartswith_lookup(self):
        tests = self.names + ['O', 'o', 'ola']
        for name in tests:
            qry = self.qry.filter(name__startswith=name)
            self.assertEqual(len(qry), len([x for x in self.names if x.startswith(name)]))

            qry = self.qry.filter(name__istartswith=name)
            self.assertEqual(len(qry), len([x for x in self.names if x.lower().startswith(name.lower())]))

    def test_regex_lookup_and_iregex_lookup(self):
        tests = [r'([A-Z])\w+', r'([A-Z])\w+\s[+]\s([A-Z])\w+', r'\-Test\-']
        for pattern in tests:
            qry = self.qry.filter(name__regex=pattern)
            self.assertEqual(len(qry), len([x for x in self.names if re.search(pattern, x)]))

            qry = self.qry.filter(name__iregex=pattern)
            self.assertEqual(len(qry), len([x for x in self.names if re.search(pattern, x, flags=re.I)]))

            # Check that the same works for ListField and SetField too
            qry = self.qry.filter(sample_list__item__regex=pattern)
            expected = [
                sample_list for sample_list in self.lists
                if any([bool(re.search(pattern, x)) for x in sample_list])
            ]
            self.assertEqual(len(qry), len(expected))

            qry = self.qry.filter(sample_list__item__iregex=pattern)
            expected = [
                sample_list for sample_list in self.lists
                if any([bool(re.search(pattern, x, flags=re.I)) for x in sample_list])
            ]
            self.assertEqual(len(qry), len(expected))

    def test_item_contains_item_icontains_lookup(self):
        tests = ['O', 'la', 'ola']
        for text in tests:
            qry = self.qry.filter(sample_list__item__contains=text)
            self.assertEqual(len(qry), 1)

            qry = self.qry.filter(sample_list__item__icontains=text)
            self.assertEqual(len(qry), 1)

    def test_item_startswith_item_istartswith_lookup(self):
        tests = ['O', 'ola', 'Ola']
        for text in tests:
            qry = self.qry.filter(sample_list__item__startswith=text)
            self.assertEqual(len(qry), 1)

            qry = self.qry.filter(sample_list__item__istartswith=text)
            self.assertEqual(len(qry), 1)

    def test_item_endswith_item_iendswith_lookup(self):
        tests = ['a', 'la', 'Ola']
        for text in tests:
            qry = self.qry.filter(sample_list__item__endswith=text)
            self.assertEqual(len(qry), 1)

            qry = self.qry.filter(sample_list__item__iendswith=text)
            self.assertEqual(len(qry), 1)


class SliceModel(models.Model):
    field1 = models.CharField(max_length=32)


class SlicingTests(TestCase):

    def test_big_slice(self):
        SliceModel.objects.create(field1="test")
        SliceModel.objects.create(field1="test2")

        self.assertFalse(
            SliceModel.objects.filter(field1__in=["test", "test2"])[9999:]
        )

        self.assertFalse(
            SliceModel.objects.filter(field1__in=["test", "test2"])[9999:10000]
        )

    def test_slicing_multi_query(self):
        SliceModel.objects.create(field1="test")
        SliceModel.objects.create(field1="test2")

        self.assertEqual(
            1,
            len(SliceModel.objects.filter(field1__in=["test", "test2"])[1:])
        )

        self.assertEqual(
            1,
            len(SliceModel.objects.filter(field1__in=["test", "test2"])[:1])
        )

        self.assertEqual(
            2,
            len(SliceModel.objects.filter(field1__in=["test", "test2"])[:2])
        )

        self.assertEqual(
            0,
            len(SliceModel.objects.filter(field1__in=["test", "test2"])[2:])
        )

    @datastore_only
    def test_slice_params_are_passed_to_query(self):
        for i in range(15):
            SliceModel.objects.create(field1=str(i))

        with sleuth.watch('google.cloud.datastore.query.Query.fetch') as Run:
            qs = SliceModel.objects.order_by("field1")[:5]

            self.assertEqual(5, len(list(qs)))
            self.assertEqual(Run.calls[0].kwargs['limit'], 5)
            self.assertEqual(Run.calls[0].kwargs['offset'], 0)

            qs = SliceModel.objects.order_by("field1")[5:]
            self.assertEqual(10, len(list(qs)))
            self.assertEqual(Run.calls[1].kwargs['limit'], None)
            self.assertEqual(Run.calls[1].kwargs['offset'], 5)

            qs = SliceModel.objects.order_by("field1")[5:10]
            self.assertEqual(5, len(list(qs)))
            self.assertEqual(Run.calls[2].kwargs['limit'], 5)
            self.assertEqual(Run.calls[2].kwargs['offset'], 5)


class NamespaceTests(TestCase):
    multi_db = True
    databases = (
        "default", "nonamespace", "non_default_connection"
    )

    @skipIf("non_default_connection" not in settings.DATABASES,
            "This test is designed for the Djangae testapp settings")
    def test_database_specific_namespaces(self):
        TestFruit.objects.create(name="Apple", color="Red")
        TestFruit.objects.create(name="Orange", color="Orange")

        TestFruit.objects.using("non_default_connection").create(name="Apple", color="Red")

        self.assertEqual(1, TestFruit.objects.using("non_default_connection").count())
        self.assertEqual(2, TestFruit.objects.count())

        with self.assertRaises(TestFruit.DoesNotExist):
            TestFruit.objects.using("non_default_connection").get(name="Orange")

        try:
            TestFruit.objects.get(name="Orange")
        except TestFruit.DoesNotExist:
            self.fail("Unable to retrieve fruit from the default namespace")

        self.assertEqual(1, TestFruit.objects.filter(name="Orange", color="Orange").count())
        self.assertEqual(0, TestFruit.objects.using("non_default_connection")
                         .filter(name="Orange", color="Orange").count())

    @skipIf("nonamespace" not in settings.DATABASES, "This test is designed for the Djangae testapp settings")
    @datastore_only
    def test_move_objects_between_namespaces(self):
        objs = [
            TestFruit.objects.create(name="Banana", color="Black"),
            TestFruit.objects.create(name="Tomato", color="Red"),
        ]
        # First, check that these objects do not exist in the other namespace.
        # We check this in several ways to check that the namespace functionality works in the
        # various different commands of the DB backend
        other_qs = TestFruit.objects.using("nonamespace")
        self.assertEqual(len(other_qs.all()), 0)
        self.assertEqual(other_qs.count(), 0)
        for obj in objs:
            self.assertRaises(TestFruit.DoesNotExist, other_qs.get, name=obj.name)
        # Now re-save both of the objects into the other namespace
        for obj in objs:
            obj.save(using="nonamespace")
        # And now check that they DO exist in that namespace
        self.assertEqual(len(other_qs.all()), 2)
        self.assertEqual(other_qs.count(), 2)
        for obj in objs:
            self.assertEqual(other_qs.get(name=obj.name), obj)
        # Now delete the objects from the original (default) namespace
        TestFruit.objects.all().delete()
        # And now make sure that they exist ONLY in the other namespace
        self.assertEqual(len(TestFruit.objects.all()), 0)
        self.assertEqual(len(other_qs.all()), 2)
        self.assertEqual(TestFruit.objects.count(), 0)
        self.assertEqual(other_qs.count(), 2)
        for obj in objs:
            self.assertRaises(TestFruit.DoesNotExist, TestFruit.objects.get, name=obj.name)
            self.assertEqual(other_qs.get(name=obj.name), obj)


class DatabaseTest(TestCase):
    multi_db = True
    databases = (
        "default", "nonamespace", "non_default_connection", "non_default_database"
    )

    def setUp(self):
        super().setUp()
        # Connections are made on-demand when an operation is performed so force
        # this so the underlying clients are created.
        [connections[conn_name].ensure_connection() for conn_name in self.databases]

    @datastore_only
    def test_database_set_for_connection_client_datastore(self):
        """Test that datastore client for each connection has the correct database set
        according to the DATABASE_ID specified in settings.

        Note that we can't test writing to a different database as the legacy
        emulator doesn't support this and we can't use the new emulator
        """
        self.assertEqual(
            connections["default"].connection._client._database,
            None,
        )
        self.assertEqual(
            connections["nonamespace"].connection._client._database,
            None,
        )
        self.assertEqual(
            connections["non_default_connection"].connection._client._database,
            None,
        )
        self.assertEqual(
            connections["non_default_database"].connection._client._database,
            "foo",
        )

    @firestore_only
    def test_database_set_for_connection_client_firestore(self):
        """Test that datastore client for each connection has the correct database set
        according to the DATABASE_ID specified in settings.

        """
        DEFAULT_FIRESTORE_DATABASE_ID = "(default)"

        self.assertEqual(
            connections["default"].connection._client._database,
            DEFAULT_FIRESTORE_DATABASE_ID,
        )
        self.assertEqual(
            connections["nonamespace"].connection._client._database,
            DEFAULT_FIRESTORE_DATABASE_ID,
        )
        self.assertEqual(
            connections["non_default_connection"].connection._client._database,
            DEFAULT_FIRESTORE_DATABASE_ID,
        )
        self.assertEqual(
            connections["non_default_database"].connection._client._database,
            "foo",
        )


class CascadeDeletionTests(TestCase):
    def test_deleting_less_than_25_items(self):
        zoo = Zoo.objects.create()

        for i in range(10):
            enclosure = Enclosure.objects.create(zoo=zoo)
            for i in range(2):
                Animal.objects.create(enclosure=enclosure)

        self.assertEqual(Animal.objects.count(), 20)

        zoo.delete()

        self.assertEqual(Enclosure.objects.count(), 0)
        self.assertEqual(Animal.objects.count(), 0)

    # see https://github.com/googleapis/google-cloud-python/issues/9921
    @skip("This test should not fail once the emulator bug is fixed")
    def test_deleting_more_than_30_items(self):
        zoo = Zoo.objects.create()

        for i in range(40):
            enclosure = Enclosure.objects.create(zoo=zoo)
            for i in range(2):
                Animal.objects.create(enclosure=enclosure)

        self.assertEqual(Animal.objects.count(), 80)

        zoo.delete()

        self.assertEqual(Enclosure.objects.count(), 0)
        self.assertEqual(Animal.objects.count(), 0)


class AsyncMultiQueryTests(TestCase):

    def test_offset_is_correctly_applied(self):
        TestUser.objects.create(username="Adam", field2="Adam", first_name="A", second_name="A")
        u2 = TestUser.objects.create(username="Bob", first_name="B", second_name="B")
        TestUser.objects.create(username="Chloe", first_name="C", second_name="C")

        qs = TestUser.objects.filter(
            Q(field2="Adam") | Q(username__in=["Adam", "Bob"])
        ).order_by("username")[1:]

        self.assertEqual(len(qs), 1)
        self.assertItemsEqual([u2], qs)

    def test_limit_is_correctly_applied(self):
        TestUser.objects.create(username="Adam", field2="Adam", first_name="A", second_name="A")
        u2 = TestUser.objects.create(username="Bob", first_name="B", second_name="B")
        TestUser.objects.create(username="Chloe", first_name="C", second_name="C")

        qs = TestUser.objects.filter(
            Q(field2="Adam") | Q(username__in=["Adam", "Bob", "Chloe"])
        ).order_by("username")[1:2]

        self.assertEqual(len(qs), 1)
        self.assertItemsEqual([u2], qs)


class UniqueQueryTest(TestCase):
    def test_entity_in_cache(self):
        with transaction.atomic():
            TestUser.objects.create(id=1, username="randy")

            with sleuth.watch("google.cloud.datastore.query.Query.fetch") as fetch:
                list(TestUser.objects.filter(pk=1))
                self.assertEqual(fetch.call_count, 0)


class FieldConversionTests(TestCase):
    def test_uuid_field_handled(self):
        instance = UUIDTestModel.objects.create()
        self.assertTrue(isinstance(instance.uuid, uuid.UUID))

        instance.refresh_from_db()
        self.assertTrue(isinstance(instance.uuid, uuid.UUID))
