import re
import html
import requests
from collections import OrderedDict

from urlexpander.core import constants

__all__ = ['get_webpage_meta',
           'get_webpage_paragraphs',
           'get_webpage_description',
           'get_webpage_title']

def _search_webpage_title(text, headers=constants.headers, **kwargs):
    title = None
    regex = re.compile('<title>(.*?)</title>', re.IGNORECASE|re.DOTALL)
    try:
        title = regex.search(text).group(1)
        title = html.unescape(title)
    except: pass
    return title

    
def _search_webpage_description(text, headers=constants.headers, **kwargs):
    desc = None
    regex =  re.compile('<meta property="og?:description" content="(.*?)>', re.IGNORECASE|re.DOTALL)
    try:
        desc = regex.search(text).group(1).rstrip('/').rstrip(' ').rstrip('"')
        desc = html.unescape(desc)
    except:
        pass
    return desc

def _search_webpage_paragraphs(text, headers=constants.headers, **kwargs):
    paragraphs = []
    try:
        paragraphs = re.findall(r'<p>(.*?)</p>', text)
        paragraphs = [html.unescape(p) for p in paragraphs]
    except:
        pass
    return paragraphs

def get_webpage_title(url, headers=constants.headers, **kwargs):
    r = requests.get(url, headers=headers, **kwargs)
    title = _search_webpage_title(r.text)
    return title

def get_webpage_description(url, headers=constants.headers, **kwargs):
    r = requests.get(url, headers=headers, **kwargs)
    desc = _search_webpage_description(r.text)
    return desc

def get_webpage_paragraphs(url, headers=constants.headers, **kwargs):
    r = requests.get(url, headers=headers, **kwargs)
    paragraphs = _search_webpage_paragraphs(r.text)
    return paragraphs

def get_webpage_meta(url, headers=constants.headers, **kwargs):
    r = requests.get(url, headers=headers, **kwargs)
    text = r.text
    meta = OrderedDict(url = url,
                       title = _search_webpage_title(text),
                       description = _search_webpage_description(text),
                       paragraphs = _search_webpage_paragraphs(text))
    return meta