from __future__ import print_function, unicode_literals, division, absolute_import
from future import standard_library
standard_library.install_aliases()  # noqa
from builtins import *  # noqa

from future.utils import viewitems  # noqa
from past.builtins import basestring  # noqa

import requests

from pugnlp import regexes

# amazon product reviews for recommendation engine training and review sentiment analysis
response = requests.get('http://jmcauley.ucsd.edu/data/amazon/')
urls_product_review = [m[0] for m in regexes.cre_url_popular.findall(response.text) if m[0].lower().endswith('.json.gz')]

response = requests.get('http://jmcauley.ucsd.edu/data/amazon/qa/')
urls_question_answer = [m[1] for m in regexes.cre_href.findall(response.text) if m[1].lower().endswith('.json.gz')]


with open('download_mccauley_autogenerated.sh', 'w') as f:
    for pr in urls_product_review:
        f.write('wget ' + pr)
    for qa in urls_question_answer:
        f.write('wget ' + qa)
