from io import StringIO, BytesIO
import os
import re
from time import sleep
import random
import chromedriver_autoinstaller
from selenium.common.exceptions import NoSuchElementException
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
import datetime
import pandas as pd
import platform
from selenium.webdriver.common.keys import Keys
# import pathlib

from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from . import const
import urllib

# current_dir = pathlib.Path(__file__).parent.absolute()

def get_data(card, save_images = False, save_dir = None):
    """Extract data from tweet card"""
    image_links = []

    try:
        username = card.find_element_by_xpath('.//span').text
    except:
        return

    try:
        handle = card.find_element_by_xpath('.//span[contains(text(), "@")]').text
    except:
        return

    try:
        postdate = card.find_element_by_xpath('.//time').get_attribute('datetime')
    except:
        return

    try:
        text = card.find_element_by_xpath('.//div[2]/div[2]/div[1]').text
    except:
        text = ""

    try:
        embedded = card.find_element_by_xpath('.//div[2]/div[2]/div[2]').text
    except:
        embedded = ""

    #text = comment + embedded

    try:
        reply_cnt = card.find_element_by_xpath('.//div[@data-testid="reply"]').text
    except:
        reply_cnt = 0

    try:
        retweet_cnt = card.find_element_by_xpath('.//div[@data-testid="retweet"]').text
    except:
        retweet_cnt = 0

    try:
        like_cnt = card.find_element_by_xpath('.//div[@data-testid="like"]').text
    except:
        like_cnt = 0

    try:
        elements = card.find_elements_by_xpath('.//div[2]/div[2]//img[contains(@src, "https://pbs.twimg.com/")]')
        for element in elements:
        	image_links.append(element.get_attribute('src'))
    except:
        image_links = []

    #if save_images == True:
    #	for image_url in image_links:
    #		save_image(image_url, image_url, save_dir)
    # handle promoted tweets

    try:
        promoted = card.find_element_by_xpath('.//div[2]/div[2]/[last()]//span').text == "Promoted"
    except:
        promoted = False
    if promoted:
        return

    # get a string of all emojis contained in the tweet
    try:
        emoji_tags = card.find_elements_by_xpath('.//img[contains(@src, "emoji")]')
    except:
        return
    emoji_list = []
    for tag in emoji_tags:
        try:
            filename = tag.get_attribute('src')
            emoji = chr(int(re.search(r'svg\/([a-z0-9]+)\.svg', filename).group(1), base=16))
        except AttributeError:
            continue
        if emoji:
            emoji_list.append(emoji)
    emojis = ' '.join(emoji_list)

    # tweet url
    try:
        element = card.find_element_by_xpath('.//a[contains(@href, "/status/")]')
        tweet_url = element.get_attribute('href')
    except:
        return

    tweet = (username, handle, postdate, text, embedded, emojis, reply_cnt, retweet_cnt, like_cnt, image_links, tweet_url)
    return tweet



def init_driver(headless=True, proxy=None, show_images=False):
    """ initiate a chromedriver instance """

    # create instance of web driver
    chromedriver_path = chromedriver_autoinstaller.install()
    options = Options()
    if headless is True:

        print("Scraping on headless mode.")
        options.add_argument('--disable-gpu')
        options.headless = True
    else:
        options.headless = False
    options.add_argument('log-level=3')
    if proxy is not None:
        options.add_argument('--proxy-server=%s' % proxy)
    if show_images == False:
    	prefs = {"profile.managed_default_content_settings.images": 2}
    	options.add_experimental_option("prefs", prefs)
    driver = webdriver.Chrome(options=options, executable_path=chromedriver_path)
    driver.set_page_load_timeout(100)
    return driver


def log_search_page(driver, start_date, end_date, lang, display_type, words, to_account, from_account, hashtag, filter_replies, proximity):
    """ Search for this query between start_date and end_date"""

    # format the <from_account>, <to_account> and <hash_tags>
    from_account = "(from%3A" + from_account + ")%20" if from_account is not None else ""
    to_account = "(to%3A" + to_account + ")%20" if to_account is not None else ""
    hash_tags = "(%23" + hashtag + ")%20" if hashtag is not None else ""

    if words is not None:
        if len(words)==1:
            words = "(" +  str(''.join(words)) + ")%20"
        else :
            words = "(" + str('%20OR%20'.join(words)) + ")%20"
    else:
        words = ""

    if lang is not None:
        lang = 'lang%3A' + lang
    else:
        lang = ""

    end_date = "until%3A" + end_date + "%20"
    start_date = "since%3A" + start_date + "%20"

    if display_type == "Latest" or display_type == "latest":
    	display_type = "&f=live"
    elif display_type == "Image" or display_type == "image":
    	display_type = "&f=image"
    else:
    	display_type = ""

    # filter replies 
    if filter_replies == True:
        filter_replies = "%20-filter%3Areplies"
    else :
        filter_replies = ""
    # proximity
    if proximity == True:
        proximity = "&lf=on" # at the end
    else : 
        proximity = ""

    path = 'https://twitter.com/search?q='+words+from_account+to_account+hash_tags+end_date+start_date+lang+filter_replies+'&src=typed_query'+display_type+proximity
    driver.get(path)
    return path


def get_last_date_from_csv(path):
    df = pd.read_csv(path)
    return datetime.datetime.strftime(max(pd.to_datetime(df["Timestamp"])), '%Y-%m-%dT%H:%M:%S.000Z')


def log_in(driver, timeout=10):
    username = const.USERNAME
    password = const.PASSWORD

    driver.get('https://www.twitter.com/login')
    username_xpath = '//input[@name="session[username_or_email]"]'
    password_xpath = '//input[@name="session[password]"]'

    username_el = WebDriverWait(driver, timeout).until(EC.presence_of_element_located((By.XPATH, username_xpath)))
    password_el = WebDriverWait(driver, timeout).until(EC.presence_of_element_located((By.XPATH, password_xpath)))

    username_el.send_keys(username)
    password_el.send_keys(password)
    password_el.send_keys(Keys.RETURN)


def keep_scroling(driver, data, writer, tweet_ids, scrolling, tweet_parsed, limit, scroll, last_position, save_images = False):
    """ scrolling function for tweets crawling"""

    save_images_dir = "/images"

    if save_images == True:
    	if not os.path.exists(save_images_dir):
    		os.mkdir(save_images_dir)

    while scrolling and tweet_parsed < limit:
        sleep(random.uniform(0.5, 1.5))
        # get the card of tweets
        page_cards = driver.find_elements_by_xpath('//div[@data-testid="tweet"]')
        for card in page_cards:
            tweet = get_data(card, save_images, save_images_dir)
            if tweet:
                # check if the tweet is unique
                tweet_id = ''.join(tweet[:-2])
                if tweet_id not in tweet_ids:
                    tweet_ids.add(tweet_id)
                    data.append(tweet)
                    last_date = str(tweet[2])
                    print("Tweet made at: " + str(last_date) + " is found.")
                    writer.writerow(tweet)
                    tweet_parsed += 1
                    if tweet_parsed >= limit:
                        break
        scroll_attempt = 0
        while tweet_parsed < limit:
            # check scroll position
            scroll += 1
            print("scroll ", scroll)
            sleep(random.uniform(0.5, 1.5))
            driver.execute_script('window.scrollTo(0, document.body.scrollHeight);')
            curr_position = driver.execute_script("return window.pageYOffset;")
            if last_position == curr_position:
                scroll_attempt += 1
                # end of scroll region
                if scroll_attempt >= 2:
                    scrolling = False
                    break
                else:
                    sleep(random.uniform(0.5, 1.5))  # attempt another scroll
            else:
                last_position = curr_position
                break
    return driver, data, writer, tweet_ids, scrolling, tweet_parsed, scroll, last_position


def get_users_follow(users, headless, follow=None, verbose=1, wait=2):
	""" get the following or followers of a list of users """

	# initiate the driver
	driver = init_driver(headless=headless)
	sleep(wait)
	# log in (the .env file should contain the username and password)
	log_in(driver)
	sleep(wait)
	# followers and following dict of each user
	follows_users = {}

	for user in users:
	    # log user page
	    print("Crawling @" + user + " "+ follow)
	    driver.get('https://twitter.com/' + user)
	    sleep(random.uniform(wait-0.5, wait+0.5))
	    # find the following or followers button
	    driver.find_element_by_xpath('//a[contains(@href,"/' + follow + '")]/span[1]/span[1]').click()
	    sleep(random.uniform(wait-0.5, wait+0.5))
	    # if the log in fails, find the new log in button and log in again.
	    if check_exists_by_link_text("Log in", driver):
	        login = driver.find_element_by_link_text("Log in")
	        sleep(random.uniform(wait-0.5, wait+0.5))
	        driver.execute_script("arguments[0].click();", login)
	        sleep(random.uniform(wait-0.5, wait+0.5))
	        driver.get('https://twitter.com/' + user)
	        sleep(random.uniform(wait-0.5, wait+0.5))
	        driver.find_element_by_xpath('//a[contains(@href,"/' + follow + '")]/span[1]/span[1]').click()
	        sleep(random.uniform(wait-0.5, wait+0.5))
	    # check if we must keep scrolling
	    scrolling = True
	    last_position = driver.execute_script("return window.pageYOffset;")
	    follows_elem = []
	    follow_ids = set()

	    while scrolling:
	        # get the card of following or followers
	        page_cards = driver.find_elements_by_xpath('//div[contains(@data-testid,"UserCell")]')
	        for card in page_cards:
	        	# get the following or followers element
	            element = card.find_element_by_xpath('.//div[1]/div[1]/div[1]//a[1]')
	            follow_elem = element.get_attribute('href')
	            # append to the list
	            follow_id = str(follow_elem)
	            follow_elem = '@' + str(follow_elem).split('/')[-1]
	            if follow_id not in follow_ids:
	            	follow_ids.add(follow_id)
	            	follows_elem.append(follow_elem)
	            if verbose:
	                print(follow_elem)
	        print("Found " + str(len(follows_elem)) + " " + follow)
	        scroll_attempt = 0
	        while True:
	            sleep(random.uniform(wait-0.5, wait+0.5))
	            driver.execute_script('window.scrollTo(0, document.body.scrollHeight);')
	            sleep(random.uniform(wait-0.5, wait+0.5))
	            curr_position = driver.execute_script("return window.pageYOffset;")
	            if last_position == curr_position:
	                scroll_attempt += 1

	                # end of scroll region
	                if scroll_attempt >= 3:
	                    scrolling = False
	                    break
	                    #return follows_elem
	                else:
	                    sleep(random.uniform(wait-0.5, wait+0.5))  # attempt another scroll
	            else:
	                last_position = curr_position
	                break

	    follows_users[user] = follows_elem

	return follows_users



def check_exists_by_link_text(text, driver):
    try:
        driver.find_element_by_link_text(text)
    except NoSuchElementException:
        return False
    return True


def dowload_images(urls, save_dir):

	for i, url_v in enumerate(urls):
		for j, url in enumerate(url_v):
			urllib.request.urlretrieve(url, save_dir + '/' + str(i+1) + '_' + str(j+1) + ".jpg")

