#!/usr/bin/python
# -*- coding: utf-8 *-*

# This file is part of Pymetrick.  The COPYRIGHT file at the top level of
# this repository contains the full copyright notices and license terms.

"""Modulo para generar sitemap"""

try:
    from pymetrick.version import __author__, __version__, __license__, __copyright__
except ImportError:
    raise ImportError(
        'Failed to find (autogenerated) version.py '
    )

__date__ = '2019-09-21'
__credits__ = ''
__text__ = 'Generador de codigo en fichero sitemap'
__file__ = 'sitemap.py'

import sys
# sin bytecode
sys.dont_write_bytecode = True
import os
import re
import logging
import logging.handlers

# For Python 3.6 and later
import urllib.request


# Available Sitemap types
SITEMAP_TYPES = ['web', 'mobile', 'news']
# General Sitemap tags
GENERAL_SITEMAP_TAGS = ['loc', 'changefreq', 'priority', 'lastmod']

# News specific tags
NEWS_SPECIFIC_TAGS = ['keywords', 'publication_date', 'stock_tickers']

# News Sitemap tags
NEWS_SITEMAP_TAGS = GENERAL_SITEMAP_TAGS + NEWS_SPECIFIC_TAGS

# Maximum number of urls in each sitemap, before next Sitemap is created
MAXURLS_PER_SITEMAP = 50000

# Suffix on a Sitemap index file
SITEINDEX_SUFFIX = '_sitemap.xml'

# Regular expressions tried for extracting URLs from access logs.
ACCESSLOG_CLF_PATTERN = re.compile(r'.+\s+"([^\s]+)\s+([^\s]+)\s+HTTP/\d+\.\d+"\s+200\s+.*')

# XML formats
GENERAL_SITEINDEX_HEADER = \
  '<?xml version="1.0" encoding="UTF-8"?>\n' \
  '<sitemapindex\n' \
  ' xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"\n' \
  ' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"\n' \
  ' xsi:schemaLocation="http://www.sitemaps.org/schemas/sitemap/0.9\n' \
  ' http://www.sitemaps.org/schemas/sitemap/0.9/' \
  'siteindex.xsd">\n'

NEWS_SITEINDEX_HEADER = \
  '<?xml version="1.0" encoding="UTF-8"?>\n' \
  '<sitemapindex\n' \
  ' xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"\n' \
  ' xmlns:news="http://www.google.com/schemas/sitemap-news/0.9"\n' \
  ' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"\n' \
  ' xsi:schemaLocation="http://www.sitemaps.org/schemas/sitemap/0.9\n' \
  ' http://www.sitemaps.org/schemas/sitemap/0.9/' \
  'siteindex.xsd">\n'

SITEINDEX_FOOTER = '</sitemapindex>\n'
SITEINDEX_ENTRY = \
  ' <sitemap>\n' \
  ' <loc>%(loc)s</loc>\n' \
  ' <lastmod>%(lastmod)s</lastmod>\n' \
  ' </sitemap>\n'
GENERAL_SITEMAP_HEADER = \
  '<?xml version="1.0" encoding="UTF-8"?>\n' \
  '<urlset\n' \
  ' xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"\n' \
  ' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"\n' \
  ' xsi:schemaLocation="http://www.sitemaps.org/schemas/sitemap/0.9\n' \
  ' http://www.sitemaps.org/schemas/sitemap/0.9/' \
  'sitemap.xsd">\n'

NEWS_SITEMAP_HEADER	= \
  '<?xml version="1.0" encoding="UTF-8"?>\n' \
  '<urlset\n' \
  ' xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"\n' \
  ' xmlns:news="http://www.google.com/schemas/sitemap-news/0.9"\n' \
  ' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"\n' \
  ' xsi:schemaLocation="http://www.sitemaps.org/schemas/sitemap/0.9\n' \
  ' http://www.sitemaps.org/schemas/sitemap/0.9/' \
  'sitemap.xsd">\n'

MOBILE_SITEMAP_HEADER = \
  '<?xml version="1.0" encoding="UTF-8" ?>\n' \
  '<urlset\n' \
  ' xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"\n' \
  ' xmlns:mobile="http://www.google.com/schemas/sitemap-mobile/1.0">\n' \
  ' <url>\n' \
  '     <loc>%(loc)s</loc>\n' \
  '     <mobile:mobile/>\n' \
  ' </url>\n' \
  '</urlset>\n'

'''
DEBUG - debug message
INFO - info message
WARNING - warn message
ERROR - error message
CRITICAL - critical message
'''
LOG_LEVEL_VALUE = {'DEBUG':10,'INFO':20,'WARNING':30,'ERROR':40,'CRITICAL':50}
if str(os.environ.get('PYMETRICK_LOG_LEVEL',None)).upper() in LOG_LEVEL_VALUE:
    LOG_LEVEL = LOG_LEVEL_VALUE[str(os.environ.get('PYMETRICK_LOG_LEVEL')).upper()]
else:
    LOG_LEVEL = 30
LOG_FILENAME = '-'.join([os.path.abspath(__file__).split(os.sep)[len(os.path.abspath(__file__).split(os.sep))-1],])[:-3]
LOG = logging.getLogger(LOG_FILENAME)

if 'LD_LIBRARY_PATH' in list(os.environ.keys()):
    # CGI environment
    sys.stdout = sys.stderr
    logging.basicConfig(stream = sys.stderr, level=LOG_LEVEL, format='%(filename)s[line:%(lineno)d] - %(levelname)s - %(message)s')
else:
    # not CGI environment
    logging.basicConfig(stream=sys.stderr)
    hdlr = logging.handlers.RotatingFileHandler(filename=LOG_FILENAME+'.log',mode='a', encoding='utf-8', maxBytes=1048576, backupCount=3)
    formatter = logging.Formatter('%(asctime)s - %(filename)s[line:%(lineno)d] - %(levelname)s - %(message)s')
    hdlr.setFormatter(formatter)
    LOG.addHandler(hdlr)
    LOG.setLevel(LOG_LEVEL)

def buscar(url=None, proxyUrl=None, proxyAuth=None, proxyPasswd=None):
    try:

        if proxyAuth is not None and proxyPasswd is not None:
            proxy = urllib.request.ProxyHandler({'http': 'http://{0}:{1}@{2}'.format(proxyAuth,proxyPasswd,proxyUrl)})
            auth = urllib.request.HTTPBasicAuthHandler()
            opener = urllib.request.build_opener(proxy, auth, urllib.request.HTTPHandler)
            urllib.request.install_opener(opener)

        if url:
            site = urllib.request.urlopen(url)
            html_string = site.read()
            site.close()
            print (html_string)

            match = re.findall(r'href=[\'"]?([^\'" >]+)', html_string)
            for n in match:
                n = n.replace('http://','').replace('https://','')
                if n.startswith(url.replace('http://','')):
                    print(n)
                elif n.startswith('/'):
                    print (n)

                # Open our local file for writing
            """local_file = open(file_name, "w" + file_mode)
                #Write to our local file
                local_file.write(f.read())
                local_file.close()"""

    #handle errors
    except urllib.request.HTTPError as e:
        print (e.code)
    except urllib.request.URLError as e:
        print (e.args)

def getgoogleurl(search,siteurl=False):
    if siteurl==False:
        return 'http://www.google.com/search?q='+urllib.request.quote(search)+'&oq='+urllib.request.quote(search)
    else:
        return 'http://www.google.com/search?q=site:'+urllib.request.quote(siteurl)+'%20'+urllib.request.quote(search)+'&oq=site:'+urllib2.quote(siteurl)+'%20'+urllib2.quote(search)

def getgooglelinks(search,siteurl=False,proxyUrl=None,proxyAuth=None,proxyPasswd=None):
   #google devolvera 403 si no se envia un user-agent valido
   headers = {'User-agent':'Mozilla/11.0'}
   if proxyAuth is not None and proxyPasswd is not None:
       proxy = urllib.request.ProxyHandler({'http': 'http://{0}:{1}@{2}'.format(proxyAuth,proxyPasswd,proxyUrl)})
       auth = urllib.request.HTTPBasicAuthHandler()
       opener = urllib.request.build_opener(proxy, auth, urllib.request.HTTPHandler)
       urllib.request.install_opener(opener)

   req = urllib.request.Request(getgoogleurl(search,siteurl),None,headers)
   site = urllib.request.urlopen(req)
   data = site.read()
   site.close()

   #no beatifulsoup because google html is generated with javascript
   start = data.find('<div id="res">')
   end = data.find('<div id="foot">')
   if data[start:end]=='':
      #error, no links to find
      return False
   else:
      links =[]
      data = data[start:end]
      start = 0
      end = 0
      while start>-1 and end>-1:
          #get only results of the provided site
          if siteurl==False:
            start = data.find('<a href="/url?q=')
          else:
            start = data.find('<a href="/url?q='+str(siteurl))
          data = data[start+len('<a href="/url?q='):]
          end = data.find('&amp;sa=U&amp;ei=')
          if start>-1 and end>-1:
              link =  urllib2.unquote(data[0:end])
              data = data[end:len(data)]
              if link.find('http')==0:
                  links.append(link)
      return links

if __name__ == "__main__":
    print ('''copyright {0}'''.format( __copyright__))
    print ('''license {0}'''.format( __license__))
    print ('''version {0}'''.format( __version__))
    if len(sys.argv) < 2:
        sys.stderr.write("for help use -h o --help")
    elif sys.argv[1]=='-h' or sys.argv[1]=='--help':
        print (b'''
        Puede cargar un texto read Structured Text que siempre debe tener una extension .rst :
        pagina = rst4html('rst_text.rst')
        posteriormente recuperara los datos html creados con :
        pagina.get()
        ''')


