# django-salesforce
#
# by Phil Christensen
# (c) 2012-2013 Freelancers Union (http://www.freelancersunion.org)
# See LICENSE.md for details
#
from decimal import Decimal
from distutils.util import strtobool
import datetime
import os
import pytz
import random
import string

from django.conf import settings
from django.db import connections
from django.db.models import Q, Avg, Count, Sum, Min, Max
from django.test import TestCase
from django.utils import timezone

from salesforce.testrunner.example.models import (Account, Contact, Lead, User,
        ApexEmailNotification, BusinessHours, ChargentOrder, CronTrigger,
        Opportunity, OpportunityContactRole,
        Product, Pricebook, PricebookEntry, Note, Task,
        Organization, models_template,
        )
from salesforce import router, DJANGO_110_PLUS, DJANGO_111_PLUS
import salesforce
from ..backend.test_helpers import skip, skipUnless, expectedFailure, expectedFailureIf # test decorators
from ..backend.test_helpers import current_user, default_is_sf, sf_alias, uid

import logging
log = logging.getLogger(__name__)

QUIET_KNOWN_BUGS = strtobool(os.getenv('QUIET_KNOWN_BUGS', 'false'))
test_email = 'test-djsf-unittests%s@example.com' % uid
sf_databases = [db for db in connections if router.is_sf_database(db)]

_sf_tables = []
def sf_tables():
    if not _sf_tables and default_is_sf:
        for x in connections[sf_alias].introspection.table_list_cache['sobjects']:
            _sf_tables.append(x['name'])
    return _sf_tables

def refresh(obj):
    """Get the same object refreshed from the same db.
    """
    db = obj._state.db
    return type(obj).objects.using(db).get(pk=obj.pk)


class BasicSOQLRoTest(TestCase):
    """Tests that need no setUp/tearDown"""

    @classmethod
    def setUpClass(cls):
        """Add contact if less than 2 exist"""
        super(BasicSOQLRoTest, cls).setUpClass()
        some_contacts = Contact.objects.all()[:2]
        if len(some_contacts) < 2:
            for i in range(2 - len(some_contacts)):
                contact = Contact(first_name='sf_test demo', last_name='Test %d' %i)
                contact.save()
        if User.objects.count() == 0:
            user = User(Username=current_user)
            user.save()

    @skipUnless(default_is_sf, "Default database should be any Salesforce.")
    def test_raw(self):
        """Get the first two contact records.

        (At least 3 manually created Contacts must exist before these read-only tests.)
        """
        contacts = Contact.objects.raw(
                "SELECT Id, LastName, FirstName FROM Contact "
                "LIMIT 2", translation={'id': 'Id'})
        self.assertEqual(len(contacts), 2)
        # It had a side effect that the same assert failed second times.
        self.assertEqual(len(contacts), 2)
        '%s' % contacts[0].__dict__  # Check that all fields are accessible

    @skipUnless(default_is_sf, "Default database should be any Salesforce.")
    def test_raw_foreignkey_id(self):
        """Get the first two contacts by raw query with a ForeignKey id field.
        """
        contacts = Contact.objects.raw(
                "SELECT Id, LastName, FirstName, OwnerId FROM Contact "
                "LIMIT 2")
        self.assertEqual(len(contacts), 2)
        '%s' % contacts[0].__dict__  # Check that all fields are accessible
        self.assertIn('@', contacts[0].owner.Email)

    def test_select_all(self):
        """Get the first two contact records.
        """
        contacts = Contact.objects.all()[0:2]
        self.assertEqual(len(contacts), 2)

    @skipUnless(default_is_sf, "Default database should be any Salesforce.")
    def test_foreign_key(self):
        """Verify that the owner of an Contact is the currently logged admin.
        """
        current_sf_user = User.objects.get(Username=current_user)
        test_contact = Contact(first_name='sf_test', last_name='my')
        test_contact.save()
        try:
            contact = Contact.objects.filter(owner=current_sf_user)[0]
            user = contact.owner
            # This user can be e.g. 'admins@freelancersunion.org.prod001'.
            self.assertEqual(user.Username, current_user)
        finally:
            test_contact.delete()

    def test_foreign_key_column(self):
        """Verify filtering by a column of related parent object.
        """
        test_account = Account(Name='sf_test account')
        test_account.save()
        test_contact = Contact(first_name='sf_test', last_name='my', account=test_account)
        test_contact.save()
        try:
            contacts = Contact.objects.filter(account__Name='sf_test account')
            self.assertEqual(len(contacts), 1)
        finally:
            test_contact.delete()
            test_account.delete()

    def test_simple_select_related(self):
        """Verify that simple_selct_related does not require additional queries.
        """
        test_account = Account(Name='sf_test account')
        test_account.save()
        test_contact = Contact(first_name='sf_test', last_name='my', account=test_account)
        test_contact.save()
        req_count_0 = salesforce.backend.driver.request_count
        try:
            qs = Contact.objects.filter(account__Name='sf_test account').simple_select_related('account')
            contacts = list(qs)
            req_count_1 = salesforce.backend.driver.request_count
            [x.account.Name for x in contacts]
            req_count_2 = salesforce.backend.driver.request_count
            self.assertEqual(req_count_1, req_count_0 + 2)
            self.assertEqual(req_count_2, req_count_1)
            self.assertGreaterEqual(len(contacts), 1)
        finally:
            test_contact.delete()
            test_account.delete()

    @skipUnless(default_is_sf, "Default database should be any Salesforce.")
    def test_one_to_one_field(self):
        # test 1a is unique field
        self.assertEqual(ApexEmailNotification._meta.get_field('user').unique, True)

        current_sf_user = User.objects.get(Username=current_user)
        orig_objects = list(ApexEmailNotification.objects.filter(
                Q(user=current_sf_user) | Q(email='apex.bugs@example.com')))
        try:
            notifier_u = current_sf_user.apex_email_notification
            new_u = None
        except ApexEmailNotification.DoesNotExist:
            notifier_u = new_u = ApexEmailNotification(user=current_sf_user)
            notifier_u.save()
        try:
            notifier_e = ApexEmailNotification.objects.get(email='apex.bugs@example.com')
            new_e = None
        except ApexEmailNotification.DoesNotExist:
            notifier_e = new_e = ApexEmailNotification(email='apex.bugs@example.com')
            notifier_e.save()

        try:
            # test 1b is unique value
            duplicate = ApexEmailNotification(user=current_sf_user)
            # the method self.assertRaise was too verbose about exception
            try:
                duplicate.save()
            except salesforce.backend.base.SalesforceError as exc:
                self.assertEqual(exc.data['errorCode'], 'DUPLICATE_VALUE')
            else:
                self.assertRaises(salesforce.backend.base.SalesforceError, duplicate.save)

            # test 2: the reverse relation is a value, not a set
            result = User.objects.exclude(apex_email_notification__user=None)
            self.assertIn(current_user, [x.Username for x in result])

            # test 3: relation to the parent
            result = ApexEmailNotification.objects.filter(user__Username=notifier_u.user.Username)
            self.assertEqual(len(result), 1)
            self.assertEqual(result[0].user_id, notifier_u.user_id)
        finally:
            if new_u:
                new_u.delete()
            if new_e:
                new_e.delete()

        # this had fail, therefore moved at the end and itemized for debugging
        # qs = User.objects.exclude(apex_email_notification=None)
        # print(qs.query.get_compiler('salesforce').as_sql())
        # list(qs)
        list(User.objects.exclude(apex_email_notification=None))

    def test_update_date(self):
        """Test updating a date.
        """
        # removed microseconds only for easy compare in the test, no problem
        now = timezone.now().replace(microsecond=0)
        contact = Contact(first_name='sf_test', last_name='my')
        contact.save()
        contact = refresh(contact)
        try:
            contact.email_bounced_date = now
            contact.save()
            self.assertEqual(refresh(contact).email_bounced_date, now)
        finally:
            contact.delete()

    def test_insert_date(self):
        """Test inserting a date.
        """
        now = timezone.now().replace(microsecond=0)
        contact = Contact(
                first_name='Joe',
                last_name='Freelancer',
                email_bounced_date=now)
        contact.save()
        try:
            self.assertEqual(refresh(contact).email_bounced_date, now)
        finally:
            contact.delete()

    @skipUnless(default_is_sf, "Default database should be any Salesforce.")
    def test_default_specified_by_sf(self):
        """Verify insert of object with a field with default value on create by SF.

        (The default is used only for a field unspecified in SF REST API, but
        not for None or any similar value. It was a pain for some unimportant
        foreign keys that don't accept null.)
        """
        # Verify a smart default is used.
        contact = Contact(first_name='sf_test', last_name='my')
        contact.save()
        try:
            self.assertEqual(refresh(contact).owner.Username, current_user)
        finally:
            contact.delete()
        # Verify that an explicit value is possible for this field.
        other_user_obj = User.objects.exclude(Username=current_user).filter(IsActive=True)[0]
        contact = Contact(first_name='sf_test', last_name='your',
                owner=other_user_obj)
        contact.save()
        try:
            self.assertEqual(
                    refresh(contact).owner.Username, other_user_obj.Username)
        finally:
            contact.delete()

    def test_not_null_related(self):
        """Verify conditions `isnull` for foreign keys: filter(Account=None)

        filter(Account__isnull=True) and nested in Q(...) | Q(...).
        """
        test_contact = Contact(first_name='sf_test', last_name='my')
        test_contact.save()
        try:
            contacts = Contact.objects.filter(Q(account__isnull=True) |
                    Q(account=None), account=None, account__isnull=True,
                    first_name='sf_test')
            self.assertEqual(len(contacts), 1)
        finally:
            test_contact.delete()

    def test_unicode(self):
        """Make sure weird unicode breaks properly.
        """
        test_lead = Lead(FirstName=u'\u2603', LastName="Unittest Unicode",
                Email='test-djsf-unicode-email@example.com',
                Company="Some company")
        test_lead.save()
        try:
            self.assertEqual(refresh(test_lead).FirstName, u'\u2603')
        finally:
            test_lead.delete()

    def test_date_comparison(self):
        """Test that date comparisons work properly.
        """
        today = datetime.datetime(2013, 8, 27)
        if settings.USE_TZ:
            today = timezone.make_aware(today, pytz.utc)
        yesterday = today - datetime.timedelta(days=1)
        tomorrow = today + datetime.timedelta(days=1)
        contact = Contact(first_name='sf_test' + uid, last_name='date',
                email_bounced_date=today)
        contact.save()
        try:
            contacts1 = Contact.objects.filter(email_bounced_date__gt=yesterday, first_name='sf_test' + uid)
            self.assertEqual(len(contacts1), 1)
            contacts2 = Contact.objects.filter(email_bounced_date__gt=tomorrow, first_name='sf_test' + uid)
            self.assertEqual(len(contacts2), 0)
        finally:
            contact.delete()

    @skipUnless(default_is_sf, "Default database should be any Salesforce.")
    def test_insert(self):
        """Create a lead record, and make sure it ends up with a valid Salesforce ID.
        """
        test_lead = Lead(FirstName="User", LastName="Unittest Inserts",
                Email='test-djsf-inserts-email@example.com',
                Company="Some company")
        test_lead.save()
        try:
            self.assertEqual(len(test_lead.pk), 18)
        finally:
            test_lead.delete()

    @skipUnless(default_is_sf, "Default database should be any Salesforce.")
    def test_double_save(self):
        """Double save without refresh of an object with a DEFAULTED_ON_CREATE
        field should not cause a problem.
        """
        oppo = Opportunity(name='test op', stage='Prospecting', close_date=datetime.date.today())
        oppo.save()
        try:
            oppo.save()
        finally:
            oppo.delete()

    def test_delete(self):
        """Create a lead record, then delete it, and make sure it's gone.
        """
        test_lead = Lead(FirstName="User", LastName="Unittest Deletes",
                Email='test-djsf-delete-email@example.com',
                Company="Some company")
        test_lead.save()
        test_lead.delete()

        self.assertRaises(Lead.DoesNotExist, Lead.objects.get, Email='test-djsf-delete-email@example.com')

    @skipUnless(default_is_sf, "Default database should be any Salesforce.")
    def test_decimal_precision(self):
        """Verify the same precision of saved and retrived DecimalField
        """
        product = Product(Name="Test Product")
        product.save()

        # The price for a product must be set in the standard price book.
        # http://www.salesforce.com/us/developer/docs/api/Content/sforce_api_objects_pricebookentry.htm
        pricebook = Pricebook.objects.get(Name="Standard Price Book")
        saved_pricebook_entry = PricebookEntry(Product2=product, Pricebook2=pricebook, UnitPrice=Decimal('1234.56'), UseStandardPrice=False)
        saved_pricebook_entry.save()
        retrieved_pricebook_entry = PricebookEntry.objects.get(pk=saved_pricebook_entry.pk)

        try:
            self.assertEqual(saved_pricebook_entry.UnitPrice, retrieved_pricebook_entry.UnitPrice)
        finally:
            retrieved_pricebook_entry.delete()
            product.delete()

    def test_simple_custom_object(self):
        """Create, read and delete a simple custom object `django_Test__c`.
        """
        from salesforce.testrunner.example.models import Attachment, Test
        if not 'django_Test__c' in sf_tables():
            self.skipTest("Not found custom object 'django_Test__c'")
        results = Test.objects.all()[0:1]
        obj = Test(test_text='sf_test')
        obj.save()
        attachment = Attachment(name='test attachment', parent=obj)
        attachment.save()
        try:
            results = Test.objects.all()[0:1]
            self.assertEqual(len(results), 1)
            self.assertEqual(results[0].test_text, 'sf_test')
        finally:
            attachment.delete()
            obj.delete()

    def test_datetime_miliseconds(self):
        """Verify that a field with milisecond resolution is readable.
        """
        triggers = CronTrigger.objects.all()
        if not triggers:
            self.skipTest("No object with milisecond resolution found.")
        self.assertTrue(isinstance(triggers[0].PreviousFireTime, datetime.datetime))
        # The reliability of this is only 99.9%, therefore it is commented out.
        #self.assertNotEqual(trigger.PreviousFireTime.microsecond, 0)

    @skipUnless(default_is_sf, "Default database should be any Salesforce.")
    def test_time_field(self):
        """Test a TimeField (read, modify, verify).
        """
        obj_orig = BusinessHours.objects.all()[0]
        obj = refresh(obj_orig)
        self.assertTrue(isinstance(obj.MondayStartTime, datetime.time))
        obj.MondayStartTime = datetime.time(23, 59)
        obj.save()
        obj = refresh(obj)
        try:
            self.assertEqual(obj.MondayStartTime, datetime.time(23, 59))
        finally:
            obj_orig.save()

    def test_account_insert_delete(self):
        """Test insert and delete an account (normal or personal SF config)
        """
        if settings.PERSON_ACCOUNT_ACTIVATED:
            test_account = Account(FirstName='IntegrationTest',
                    LastName='Account')
        else:
            test_account = Account(Name='IntegrationTest Account')
        test_account.save()
        try:
            accounts = Account.objects.filter(Name='IntegrationTest Account')
            self.assertEqual(len(accounts), 1)
        finally:
            test_account.delete()

    def test_similarity_filter_operators(self):
        """Test filter operators that use LIKE 'something%' and similar.
        """
        User.objects.get(Username__exact=current_user)
        User.objects.get(Username__iexact=current_user.upper())
        User.objects.get(Username__contains=current_user[1:-1])
        User.objects.get(Username__icontains=current_user[1:-1].upper())
        User.objects.get(Username__startswith=current_user[:-1])
        User.objects.get(Username__istartswith=current_user[:-1].upper())
        User.objects.get(Username__endswith=current_user[1:])
        User.objects.get(Username__iendswith=current_user[1:].upper())
        # Operators regex and iregex not tested because they are not supported.

    @skipUnless(default_is_sf, "Default database should be any Salesforce.")
    def test_bulk_create(self):
        """Create two Contacts by one request in one command and find them.
        """
        account = Account(Name='test bulk')
        account.save()
        try:
            objects = [Contact(last_name='sf_test a', account=account),
                       Contact(last_name='sf_test b', account=account)]
            request_count_0 = salesforce.backend.driver.request_count

            ret = Contact.objects.bulk_create(objects)

            request_count_1 = salesforce.backend.driver.request_count
            self.assertEqual(request_count_1,  request_count_0 + 1)
            self.assertEqual(len(Contact.objects.filter(account=account)), 2)
        finally:
            account.delete()

    @expectedFailureIf(QUIET_KNOWN_BUGS and DJANGO_111_PLUS)
    def test_bulk_update(self):
        """Create two Contacts by one request in one command, find them.
        """
        account_0, account_1 = [Account(Name='test' + uid), Account(Name='test' + uid)]
        account_0.save()
        account_1.save()
        try:
            request_count_0 = salesforce.backend.driver.request_count
            Account.objects.filter(pk=account_0.pk).update(Name="test2" + uid)
            Account.objects.filter(pk__in=[account_1.pk]).update(Name="test2" + uid)
            Account.objects.filter(pk__in=Account.objects.filter(Name='test2' + uid)).update(Name="test3" + uid)
            request_count_1 = salesforce.backend.driver.request_count
            self.assertEqual(Account.objects.filter(Name='test3' + uid).count(), 2)
            self.assertEqual(request_count_1,  request_count_0 + 4)
        finally:
            account_0.delete()
            account_1.delete()

    def test_escape_single_quote(self):
        """Test single quotes in strings used in a filter

        Verity that they are escaped properly.
        """
        account_name = '''Dr. Evil's Giant\\' "Laser", LLC'''
        account = Account(Name=account_name)
        account.save()
        try:
            self.assertTrue(Account.objects.filter(Name=account_name).exists())
        finally:
            account.delete()

    @skipUnless(default_is_sf, "Default database should be any Salesforce.")
    def test_escape_single_quote_in_raw_query(self):
        """Test that manual escaping within a raw query is not double escaped.
        """
        account_name = '''Dr. Evil's Giant\\' "Laser", LLC'''
        account = Account(Name=account_name)
        account.save()

        manually_escaped = '''Dr. Evil\\'s Giant\\\\\\' "Laser", LLC'''
        try:
            retrieved_account = Account.objects.raw(
                "SELECT Id, Name FROM Account WHERE Name = '%s'" % manually_escaped)[0]
            self.assertEqual(account_name, retrieved_account.Name)
        finally:
            account.delete()

    def test_raw_query_empty(self):
        """Test that the raw query works even for queries with empty results.

        Raw queries with supported empty results are an improvement over normal
        Django. It is useful if some unimplemented features
        of django-salesforce are solved by writing a raw query.
        """
        len(list(Contact.objects.raw("SELECT Id, FirstName FROM Contact WHERE FirstName='nonsense'")))

    def test_range_simple(self):
        """Test simple range filters".
        """
        qs = Contact.objects.filter(Q(name__range=('c', 'e')))
        soql = qs.query.get_compiler('salesforce').as_sql()[0]
        self.assertIn(u"(Contact.Name >= %s AND Contact.Name <= %s)", soql)
        len(qs)

    def test_range_combined(self):
        """Test combined filters "a OR b AND c".
        """
        qs = Contact.objects.filter(Q(name='a') | Q(name__range=('c', 'e')))
        soql = qs.query.get_compiler('salesforce').as_sql()[0]
        self.assertIn(u"Contact.Name = %s OR (Contact.Name >= %s AND Contact.Name <= %s)", soql)
        len(qs)

    def test_range_lookup(self):
        """Get the test opportunity record by range condition.
        """
        test_opportunity = Opportunity(
                        name="Example Opportunity",
                        close_date=datetime.date(year=2015, month=7, day=30),
                        stage="Prospecting",
                        amount=130000.00
        )
        test_opportunity.save()
        try:
            # Test date objects
            start_date = datetime.date(year=2015, month=7, day=29)
            end_date = datetime.date(year=2015, month=8, day=1)
            oppy = Opportunity.objects.filter(close_date__range=(start_date, end_date))[0]
            self.assertEqual(oppy.name, 'Example Opportunity')
            self.assertEqual(oppy.stage, 'Prospecting')

            # Test datetime objects (now +- 10 minutes for clock inaccuracy)
            start_time = timezone.now() - datetime.timedelta(seconds=600)
            end_time = timezone.now() + datetime.timedelta(seconds=600)
            opportunities = Opportunity.objects.filter(created_date__range=(start_time, end_time))[:1]
            self.assertEqual(len(opportunities), 1, "Failed range filter or maybe incorrectly set clock")
            oppy = opportunities[0]
            self.assertEqual(oppy.name, 'Example Opportunity')
            self.assertEqual(oppy.stage, 'Prospecting')

            # Test DecimalField
            low_amount, high_amount = 100000.00, 140000.00
            oppy = Opportunity.objects.filter(amount__range=(low_amount, high_amount))[0]
            self.assertEqual(oppy.amount, 130000.00)

            # Test CharField
            low_letter, high_letter = 'E', 'G'
            oppy = Opportunity.objects.filter(name__range=(low_letter, high_letter))[0]
            self.assertEqual(oppy.name, 'Example Opportunity')
        finally:
            test_opportunity.delete()

    def test_combined_international(self):
        """Test combined filters with international characters.
        """
        # This is OK for long time
        len(Contact.objects.filter(Q(first_name=u'\xe1') & Q(last_name=u'\xe9')))
        # This was recently fixed
        len(Contact.objects.filter(Q(first_name=u'\xe1') | Q(last_name=u'\xe9')))
        len(Contact.objects.filter(Q(first_name='\xc3\xa1') | Q(last_name='\xc3\xa9')))

    @skipUnless(default_is_sf, "Default database should be any Salesforce.")
    def test_aggregate_query(self):
        """Test for different aggregate function.
        """
        test_product = Product(Name='test soap')
        test_product.save()
        test_product2 = Product(Name='test brush')
        test_product2.save()
        pricebook = Pricebook.objects.get(Name="Standard Price Book")
        PricebookEntry(Product2=test_product, Pricebook2=pricebook,
                UseStandardPrice=False, UnitPrice=Decimal(100)).save()
        PricebookEntry(Product2=test_product2, Pricebook2=pricebook,
                UseStandardPrice=False, UnitPrice=Decimal(80)).save()
        try:
            x_products = PricebookEntry.objects.filter(Name__startswith='test ')
            result = x_products.aggregate(Sum('UnitPrice'), Count('UnitPrice'), Avg('UnitPrice'), Min('UnitPrice'))
            self.assertDictEqual(result, {'UnitPrice__sum': 180, 'UnitPrice__count': 2, 'UnitPrice__avg': 90.0, 'UnitPrice__min': 80})
        finally:
            # dependent PricebookEntries are just deleted automatically by SF
            test_product.delete()
            test_product2.delete()

    @skipUnless(default_is_sf, "Default database should be any Salesforce.")
    def test_z_big_query(self):
        """Test a big query that will be splitted to more requests.

        Test it as late as possible when
        """
        all_leads = Lead.objects.query_all()
        leads_list = list(all_leads)
        if all_leads.query.first_chunk_len == len(leads_list):
            self.assertLessEqual(len(leads_list), 2000, "Obsoleted constants")
            log.info("Not enough Leads accumulated (currently %d including deleted) "
                    "in the last two weeks that are necessary for splitting the "
                    "query into more requests. Number 1001 or 2001 is enough.",
                    len(leads_list))
            self.skipTest("Not enough Leads found for big query test")

    @skipUnless(default_is_sf, "Default database should be any Salesforce.")
    def test_cursor_execute_fetch(self):
        """Get results by cursor.execute(...); fetchone(), fetchmany(), fetchall()
        """
        # TODO hy: fix for concurrency
        sql = "SELECT Id, LastName, FirstName, OwnerId FROM Contact LIMIT 2"
        cursor = connections[sf_alias].cursor()
        cursor.execute(sql)
        contacts = cursor.fetchall()
        self.assertEqual(len(contacts), 2)
        self.assertIn('OwnerId', contacts[0])
        cursor.execute(sql)
        self.assertEqual(cursor.fetchone(), contacts[0])
        self.assertEqual(cursor.fetchmany(), contacts[1:])

    @skipUnless(default_is_sf, "Default database should be any Salesforce.")
    def test_cursor_execute_aggregate(self):
        """Verify that aggregate queries can be executed directly by SOQL.
        """
        # Field 'Id' is very useful for COUNT over non empty fields.
        sql = "SELECT LastName, COUNT(Id) FROM Contact GROUP BY LastName LIMIT 2"
        cursor = connections[sf_alias].cursor()
        cursor.execute(sql)
        contact_aggregate = cursor.fetchone()
        self.assertTrue('LastName' in contact_aggregate)
        self.assertGreaterEqual(contact_aggregate['expr0'], 1)

    @skipUnless(default_is_sf, "Default database should be any Salesforce.")
    def test_errors(self):
        """Test for improving code coverage.
        """
        # broken query raises exception
        bad_queryset = Lead.objects.raw("select XYZ from Lead")
        bad_queryset.query.debug_silent = True
        self.assertRaises(salesforce.backend.base.SalesforceError, list, bad_queryset)

    @expectedFailureIf(QUIET_KNOWN_BUGS)
    def test_queryset_values(self):
        """Test list of dict qs.values() and list of tuples qs.values_list()
        """
        tmp = Contact.objects.values('pk', 'last_name')
        # import pdb; pdb.set_trace()
        tmp[0]
        values = Contact.objects.values()[:2]
        self.assertEqual(len(values), 2)
        self.assertIn('first_name', values[0])
        self.assertGreater(len(values[0]), 3)

        values = Contact.objects.values('pk', 'first_name', 'last_name')[:2]
        self.assertEqual(len(values), 2)
        self.assertIn('first_name', values[0])
        self.assertNotIn('attributes', values[0])
        self.assertEqual(len(values[0]), 3)
        if default_is_sf:
            self.assertRegexpMatches(values[0]['pk'], '^003')

        values_list = Contact.objects.values_list('pk', 'first_name', 'last_name')[:2]
        self.assertEqual(len(values_list), 2)
        v0 = values[0]
        self.assertEqual(values_list[0], (v0['pk'], v0['first_name'], v0['last_name']))

    @skipUnless(default_is_sf, "Default database should be any Salesforce.")
    def test_double_delete(self):
        """Test that repeated delete of the same object is ignored

        the same way like "DELETE FROM Contact WHERE Id='deleted yet'" would do.
        """
        contact = Contact(last_name='sf_test',
                owner=User.objects.get(Username=current_user))
        contact.save()
        contact_id = contact.pk
        Contact(pk=contact_id).delete()
        # Id of a deleted object or a too small valid Id shouldn't raise
        Contact(pk=contact_id).delete()
        # Simulate the same with obsoleted oauth session
        # It is not possible to use salesforce.auth.expire_token() to simulate
        # expiration because it forces reauhentication before the next request
        salesforce.auth.oauth_data[sf_alias]['access_token'] = '* something invalid *'
        Contact(pk=contact_id).delete()
        # Id of completely deleted item or fake but valid item.
        Contact(pk='003000000000000AAA').delete()
        #bad_id = '003000000000000AAB' # Id with an incorrect uppercase mask
        #self.assertRaises(salesforce.backend.base.SalesforceError, Contact(pk=bad_id).delete)

    @skipUnless(default_is_sf, "Default database should be any Salesforce.")
    @skipUnless(len(sf_databases) > 1, "Only one SF database found.")
    def test_multiple_sf_databases(self):
        """Test a connection to two sf sandboxes of the same organization.
        """
        other_db = [db for db in sf_databases if db != sf_alias][0]
        c1 = Contact(last_name='sf_test 1')
        c2 = Contact(last_name='sf_test 2')
        c1.save()
        c2.save(using=other_db)
        try:
            user1 = refresh(c1).owner
            user2 = refresh(c2).owner
            username1 = user1.Username
            username2 = user2.Username
            # Verify different usernames, like it is usual in sandboxes
            self.assertNotEqual(user1._state.db, user2._state.db)
            self.assertNotEqual(username1, username2)
            expected_user2 = connections[other_db].settings_dict['USER']
            self.assertEqual(username1, current_user)
            self.assertEqual(username2, expected_user2)
        finally:
            c1.delete()
            c2.delete()

    @skipUnless(default_is_sf, "Default database should be any Salesforce.")
    def test_expired_auth_id(self):
        """Test the code for expired auth ID for multiple SF databases.

        No similar test exists for a single db.
        """
        self.assertGreaterEqual(len(sf_databases), 1)
        objects = []
        for db in sf_databases:
            c = Contact(last_name='sf_test %s' % db)
            c.save(using=db)
            objects.append(c)
        try:
            # simulate that a request with invalid/expired auth ID re-authenticates
            # and succeeds.
            for db in sf_databases:
                salesforce.auth.oauth_data[db]['access_token'] += 'simulated invalid/expired'
            for x in objects:
                self.assertTrue(refresh(x))
        finally:
            for x in objects:
                x.delete()

    # This should not be implemented due to Django conventions.
    #def test_raw_aggregate(self):
    #   # raises "TypeError: list indices must be integers, not str"
    #   list(Contact.objects.raw("select Count() from Contact"))

    @skipUnless(default_is_sf, "Default database should be any Salesforce.")
    @expectedFailureIf(QUIET_KNOWN_BUGS and DJANGO_110_PLUS)
    def test_only_fields(self):
        """Verify that access to "only" fields doesn't require a request, but others do.
        """
        def assert_n_requests(n):
            # verify the necessary number of requests
            # print("-- req=%d (expect %d)" % (salesforce.backend.driver.request_count - request_count_0, n))
            self.assertEqual(salesforce.backend.driver.request_count - request_count_0, n)

        # print([(x.id, x.last_name) for x in Contact.objects.only('last_name').order_by('id')[:2]])
        request_count_0 = salesforce.backend.driver.request_count
        sql = User.objects.only('Username').query.get_compiler('salesforce').as_sql()[0]
        self.assertEqual(sql, 'SELECT User.Id, User.Username FROM User')
        assert_n_requests(0)

        user = User.objects.only('Username').order_by('pk')[0]                  # req
        assert_n_requests(1)

        # Verify that deferred fields work
        user.pk                                                                 # no request
        user_username = user.Username
        self.assertIn('@', user_username)
        assert_n_requests(1)

        self.assertGreater(len(user.LastName), 0)                               # req
        #import pdb; pdb.set_trace()
        assert_n_requests(2)

        self.assertEqual(user_username, User.objects.get(pk=user.pk).Username)  # req
        assert_n_requests(3)

        xx = Contact.objects.only('last_name').order_by('pk')[0]                # req
        assert_n_requests(4)

        xx.last_name                                                            # no req
        # print((xx.id, xx.last_name))
        assert_n_requests(4)

        xy = Contact.objects.only('account').order_by('pk')[1]                  # req
        assert_n_requests(5)

        xy.account_id                                                           # no req
        assert_n_requests(5)

        # print((xy.id, xy.last_name))                                          # req
        #import pdb; pdb.set_trace()
        self.assertGreater(len(xy.last_name), 0)
        assert_n_requests(6)

        xy.last_name                                                            # no req
        assert_n_requests(6)                                                    # total 6 requests

    @skipUnless(default_is_sf, "Default database should be any Salesforce.")
    @expectedFailureIf(QUIET_KNOWN_BUGS and DJANGO_110_PLUS)
    def test_defer_fields(self):
        """Verify that access to a deferred field requires a new request, but others don't.
        """
        request_count_0 = salesforce.backend.driver.request_count
        contact = Contact.objects.defer('email')[0]
        self.assertEqual(salesforce.backend.driver.request_count, request_count_0 + 1)
        _ = contact.last_name
        self.assertEqual(salesforce.backend.driver.request_count, request_count_0 + 1)
        _ = contact.email
        self.assertEqual(salesforce.backend.driver.request_count, request_count_0 + 2)

    @expectedFailureIf(QUIET_KNOWN_BUGS and DJANGO_110_PLUS)
    def test_incomplete_raw(self):
        last_name = Contact.objects.raw("select id from Contact")[0].last_name
        self.assertGreater(len(last_name), 0)

    @skipUnless(default_is_sf, "Default database should be any Salesforce.")
    def test_filter_by_more_fk_to_the_same_model(self):
        """Test a filter with more relations to the same model.

        Verify that aliases are correctly decoded in the query compiler.
        """
        test_lead = Lead(Company='sf_test lead', LastName='name')
        test_lead.save()
        try:
            qs = Lead.objects.filter(pk=test_lead.pk,
                    owner__Username=current_user,
                    last_modified_by__Username=current_user)
            # Verify that a coplicated analysis is not performed on old Django
            # so that the query can be at least somehow simply compiled to SOQL
            # without exceptions, in order to prevent regressions.
            sql, params = qs.query.get_compiler('salesforce').as_sql()
            # Verify expected filters in SOQL compiled by new Django
            self.assertIn('Lead.Owner.Username = %s', sql)
            self.assertIn('Lead.LastModifiedBy.Username = %s', sql)
            # verify validity for SFDC, verify results
            refreshed_lead = qs.get()
            self.assertEqual(refreshed_lead.pk, test_lead.pk)
        finally:
            test_lead.delete()

    @skipUnless(default_is_sf, "Default database should be any Salesforce.")
    def test_filter_by_more_fk_to_the_same_model_subquery(self):
        """Test a filter with more relations to the same model.

        Verify that aliases are correctly decoded in the query compiler.
        """
        test_lead = Lead(Company='sf_test lead', LastName='name')
        test_lead.save()
        test_task = Task(who=test_lead)
        test_task.save()
        try:
            qs = Task.objects.filter(who__in=Lead.objects.filter(pk=test_lead.pk,
                    owner__Username=current_user,
                    last_modified_by__Username=current_user))
            sql, params = qs.query.get_compiler('salesforce').as_sql()
            self.assertRegexpMatches(sql, 'SELECT Task.Id, .* FROM Task WHERE Task.WhoId IN \(SELECT ')
            self.assertIn('Lead.Owner.Username = %s', sql)
            self.assertIn('Lead.LastModifiedBy.Username = %s', sql)
            refreshed_lead = qs[:1]
            self.assertEqual(len(refreshed_lead), 1)
        finally:
            test_task.delete()
            test_lead.delete()

    def test_many2many_relationship(self):
        """Verify that the related set of Many2Many relationship works

        Test for issue #55
        """
        contact = Contact.objects.all()[0]
        oppo = Opportunity(name='test op', stage='Prospecting', close_date=datetime.date.today())
        oppo.save()
        oc = OpportunityContactRole(opportunity=oppo, contact=contact, role='sponsor')
        oc.save()
        oc2 = OpportunityContactRole(opportunity=oppo, contact=contact, role='evaluator')
        oc2.save()
        try:
            qs = contact.opportunities.all()
            sql, params = qs.query.get_compiler('salesforce').as_sql()
            self.assertRegexpMatches(sql, 'SELECT .*OpportunityContactRole\.Opportunity\.StageName.* '
                    'FROM OpportunityContactRole WHERE OpportunityContactRole.ContactId =')
            self.assertEqual([x.pk for x in qs], 2 * [oppo.pk])
        finally:
            oc2.delete()
            oc.delete()
            oppo.delete()

    def test_filter_custom(self):
        """Verify that relations between custom and builtin objects

        are correctly compiled. (__r, __c etc.)
        """
        from salesforce.testrunner.example.models import Attachment, Test
        if not 'django_Test__c' in sf_tables():
            self.skipTest("Not found custom object 'django_Test__c'")
        qs = Attachment.objects.filter(parent__name='abc')
        # "SELECT Attachment.Id FROM Attachment WHERE Attachment.Parent.Name = 'abc'"
        list(qs)
        qs = Test.objects.filter(contact__last_name='Johnson')
        # "SELECT ... FROM django_Test__c WHERE django_Test__c.Contact__r.LastName = 'Johnson'")
        list(qs)
        qs = Attachment.objects.filter(parent__in=Test.objects.filter(contact__last_name='Johnson'))
        list(qs)

    def test_using_none(self):
        alias = getattr(settings, 'SALESFORCE_DB_ALIAS', 'salesforce')
        self.assertEqual(Contact.objects.using(None)._db, alias)

    @skipUnless(hasattr(models_template, 'Organization'), "Skipped because models_template.Organization doesn't exist")
    def test_dynamic_fields(self):
        """Test that fields can be copied dynamically from other model"""
        self.assertGreater(Organization.objects.get().created_by.Username, '')

# ============= Tests that need setUp Lead ==================


class BasicLeadSOQLTest(TestCase):
    """Tests that use a test Lead"""

    def setUp(self):
        """Create our test lead record.
        """
        def add_obj(obj):
            obj.save()
            self.objs.append(obj)
        #
        self.test_lead = Lead(
            FirstName="User" + uid,
            LastName="Unittest General",
            Email=test_email,
            Status='Open',
            Company="Some company, Ltd.",
        )
        self.objs = []
        self.test_lead.save()
        # This is only for demonstration that some test can be run even with
        # non SFDC database SALESFORCE_DB_ALIAS, even if the test expects some
        # contacts and the current user, but most of tests can pass only on SFDC.
        if not default_is_sf:
            add_obj(Contact(last_name='Test contact 1'))
            add_obj(Contact(last_name='Test contact 2'))
            add_obj(User(Username=current_user))

    def tearDown(self):
        """Clean up our test records.
        """
        if self.test_lead.pk is not None:
            self.test_lead.delete()
        for obj in self.objs:
            if obj.pk is not None:
                obj.delete()
        self.objs = []


    def test_exclude_query_construction(self):
        """Test that exclude query construction returns valid SOQL.
        """
        contacts = Contact.objects.filter(first_name__isnull=False
                ).exclude(email="steve@apple.com", last_name="Wozniak").exclude(last_name="smith")
        number_of_contacts = contacts.count()
        self.assertIsInstance(number_of_contacts, int)
        # the default self.test_lead shouldn't be excluded by only one nondition
        leads = Lead.objects.exclude(Email="steve@apple.com", LastName="Unittest General"
                ).filter(FirstName="User" + uid, LastName="Unittest General")
        self.assertEqual(leads.count(), 1)

    def test_get(self):
        """Get the test lead record.
        """
        lead = Lead.objects.get(Email=test_email)
        self.assertEqual(lead.FirstName, 'User' + uid)
        self.assertEqual(lead.LastName, 'Unittest General')
        if not default_is_sf:
            self.skipTest("Default database should be any Salesforce.")
        # test a read only field (formula of full name)
        self.assertEqual(lead.Name, 'User%s Unittest General' % uid)

    def test_not_null(self):
        """Get the test lead record by isnull condition.
        """
        lead = Lead.objects.get(Email__isnull=False, FirstName='User' + uid)
        self.assertEqual(lead.FirstName, 'User' + uid)
        self.assertEqual(lead.LastName, 'Unittest General')

    def test_update(self):
        """Update the test lead record.
        """
        test_lead = Lead.objects.get(Email=test_email)
        self.assertEqual(test_lead.FirstName, 'User' + uid)
        test_lead.FirstName = 'Tested'
        test_lead.save()
        self.assertEqual(refresh(test_lead).FirstName, 'Tested')

    def test_save_update_fields(self):
        """Test the save method with parameter `update_fields`

        that updates only required fields.
        """
        company_orig = self.test_lead.Company
        self.test_lead.Company = 'nonsense'
        self.test_lead.FirstName = 'John'
        self.test_lead.save(update_fields=['FirstName'])
        test_lead = refresh(self.test_lead)
        self.assertEqual(test_lead.FirstName, 'John')
        self.assertEqual(test_lead.Company, company_orig)

    def test_query_all_deleted(self):
        """Test query for deleted objects (queryAll resource).
        """
        self.test_lead.delete()
        # TODO optimize counting because this can load thousands of records
        count_deleted = Lead.objects.db_manager(sf_alias).query_all(
                ).filter(IsDeleted=True, LastName="Unittest General").count()
        if not default_is_sf:
            self.skipTest("Default database should be any Salesforce.")
        self.assertGreaterEqual(count_deleted, 1)
        count_deleted2 = Lead.objects.filter(IsDeleted=True, LastName="Unittest General").query_all().count()
        self.assertGreaterEqual(count_deleted2, count_deleted)
        self.test_lead.save()  # save anything again to be cleaned finally

    @skipUnless(default_is_sf, "Default database should be any Salesforce.")
    def test_generic_type_field(self):
        """Test that a generic foreign key can be filtered by type name and
        the type name can be referenced.
        """
        test_contact = Contact(first_name='sf_test', last_name='my')
        test_contact.save()
        note_1 = Note(title='note for Lead', parent_id=self.test_lead.pk)
        note_2 = Note(title='note for Contact', parent_id=test_contact.pk)
        note_1.save()
        note_2.save()
        try:
            self.assertEqual(Note.objects.filter(parent_type='Contact')[0].parent_type, 'Contact')
            self.assertEqual(Note.objects.filter(parent_type='Lead')[0].parent_type, 'Lead')

            note = Note.objects.filter(parent_type='Contact')[0]
            parent_model = getattr(salesforce.testrunner.example.models, note.parent_type)
            parent_object = parent_model.objects.get(pk=note.parent_id)
            self.assertEqual(parent_object.pk, note.parent_id)
        finally:
            note_1.delete()
            note_2.delete()
            test_contact.delete()
