# Scraper and Back Scraper for New York Appellate Term 1st Dept.
# CourtID: nyappterm_1st
# Court Short Name: NY
# Author: Andrei Chelaru
# Reviewer: mlr
# Date: 2015-10-30

from datetime import date, timedelta, datetime
from lxml import html
import os
from dateutil.rrule import DAILY, rrule
import re

from selenium import webdriver
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC

from juriscraper.AbstractSite import logger
from juriscraper.OpinionSite import OpinionSite
from juriscraper.lib.cookie_utils import normalize_cookies
from juriscraper.lib.network_utils import add_delay


class Site(OpinionSite):

    def __init__(self, *args, **kwargs):
        super(Site, self).__init__(*args, **kwargs)
        self.court = 'Appellate+Term,+1st+Dept'
        self.interval = 365
        self.back_scrape_iterable = [i.date() for i in rrule(
            DAILY,
            interval=self.interval,
            dtstart=date(2004, 1, 1),
            until=date(2016, 1, 1),
        )]

        self.url = 'http://iapps.courts.state.ny.us/lawReporting/Search'

        self.parameters = {
            'rbOpinionMotion': 'opinion',
            'Pty': '',
            'and_or': 'and',
            'dtStartDate': (date.today() - timedelta(days=30)).strftime("%m/%d/%Y"),
            'dtEndDate': date.today().strftime("%m/%d/%Y"),
            'court': self.court,
            'docket': '',
            'judge': '',
            'slipYear': '',
            'slipNo': '',
            'OffVol': '',
            'OffPage': '',
            'fullText': '',
            'and_or2': 'and',
            'Submit': 'Find',
            'hidden1': '',
            'hidden2': ''

        }

        self.court_id = self.__module__
        self.method = 'POST'
        self.base_path = '//tr[td[5]//a][td[7][contains(., "Opinion")]]'  # Any element with a link on the 5th column
        self.download_regex = re.compile("funcNewWindow\('(.*\.htm)'\)")
        self.use_sessions = True
        self.uses_selenium = True

    def _download(self, request_dict={}):
        """
        We use selenium to get the cookies, and then we check if we got the
        correct page. If not we retry for a total of 11 times.
        """
        if self.method == 'LOCAL':
            return super(Site, self)._download(request_dict)

        self.set_cookies()
        logger.info("Using cookies: %s" % self.cookies)
        request_dict.update({'cookies': self.cookies})

        html__ = super(Site, self)._download(request_dict)
        i = 0
        while not html__.xpath('//table') and i < 10:
            add_delay(20, 5)
            html__ = super(Site, self)._download(request_dict)
            i += 1
            logger.info("Got a bad response {} time(s)".format(i))
        return html__

    def _get_case_names(self):
        case_names = []
        for element in self.html.xpath(self.base_path):
            case_names.append(''.join(x.strip() for x in element.xpath('./td[1]//text()')))
        return case_names

    def _get_download_urls(self):
        download_urls = []
        for element in self.html.xpath(self.base_path):
            download_urls.append(''.join(self.download_regex.findall(x)[0] for x in element.xpath('./td[5]//@href')))
        return download_urls

    def _get_case_dates(self):
        case_dates = []
        for element in self.html.xpath("{}/td[2]//text()".format(self.base_path)):
            case_dates.append(datetime.strptime(element.strip(), '%m/%d/%Y'))
        return case_dates

    def _get_precedential_statuses(self):
        return ['Published'] * len(self.case_names)

    def _get_docket_numbers(self):
        docket_numbers = []
        for element in self.html.xpath(self.base_path):
            docket_numbers.append(''.join(x.strip() for x in element.xpath('./td[5]//text()')))
        return docket_numbers

    def _get_judges(self):
        judges = []
        for element in self.html.xpath('{}/td[6]'.format(self.base_path, )):
            judges.append(''.join(x.strip() for x in element.xpath('.//text()')))
        return judges

    def _get_neutral_citations(self):
        neutral_citations = []
        for element in self.html.xpath('{}/td[4]'.format(self.base_path, )):
            neutral_citations.append(''.join(x.strip() for x in element.xpath('.//text()')))
        return neutral_citations

    def _download_backwards(self, d):
        self.crawl_date = d
        self.method = 'POST'
        self.parameters = {
            'rbOpinionMotion': 'opinion',
            'Pty': '',
            'and_or': 'and',
            'dtStartDate': (d - timedelta(days=self.interval)).strftime("%m/%d/%Y"),
            'dtEndDate': d.strftime("%m/%d/%Y"),
            'court': self.court,
            'docket': '',
            'judge': '',
            'slipYear': '',
            'slipNo': '',
            'OffVol': '',
            'OffPage': '',
            'fullText': '',
            'and_or2': 'and',
            'Submit': 'Find',
            'hidden1': '',
            'hidden2': ''
        }
        self.html = self._download()
        if self.html is not None:
            # Setting status is important because it prevents the download
            # function from being run a second time by the parse method.
            self.status = 200

    def set_cookies(self):
        """Hit the main URL, and get the cookies so we can use them elsewhere.

        This gets around some of their throttling mechanisms.
        """
        logger.info("Running Selenium browser PhantomJS to get the cookies...")
        add_delay(20, 5)
        driver = webdriver.PhantomJS(
            executable_path='/usr/local/phantomjs/phantomjs',
            service_log_path=os.path.devnull,  # Disable ghostdriver.log
        )

        driver.set_window_size(1920, 1080)
        driver.get(self.url)
        WebDriverWait(driver, 30).until(
            EC.presence_of_element_located((By.NAME, "dtEndDate"))
        )
        self.cookies = normalize_cookies(driver.get_cookies())
        driver.close()

    @staticmethod
    def cleanup_content(content):
        tree = html.fromstring(content)
        core_element = tree.xpath('//partyblock')[0]
        return html.tostring(
            core_element,
            pretty_print=True,
            encoding='unicode'
        )
