import requests
import json
import csv
from bs4 import BeautifulSoup
import traceback

def get_cars(make="BMW", model="5 SERIES", postcode="SW1A 0AA", radius=1500, min_year=1995, max_year=1995, include_writeoff="include", max_attempts_per_page=5, verbose=False):

	# Basic variables

	results = []
	n_this_year_results = 0

	url = "https://www.autotrader.co.uk/results-car-search"

	keywords = {}
	keywords["mileage"] = ["miles"]
	keywords["BHP"] = ["BHP"]
	keywords["transmission"] = ["Automatic", "Manual"]
	keywords["fuel"] = ["Petrol", "Diesel", "Electric", "Hybrid – Diesel/Electric Plug-in", "Hybrid – Petrol/Electric", "Hybrid – Petrol/Electric Plug-in"]
	keywords["owners"] = ["owners"]
	keywords["body"] = ["Coupe", "Convertible", "Estate", "Hatchback", "MPV", "Pickup", "SUV", "Saloon"]
	keywords["ULEZ"] = ["ULEZ"]
	keywords["year"] = [" reg)"]
	keywords["engine"] = ["engine"]

	# Set up parameters for query to autotrader.co.uk

	params = {
		"sort": "relevance",
		"postcode": postcode,
		"radius": radius,
		"make": make,
		"model": model,
		"search-results-price-type": "total-price",
		"search-results-year": "select-year",
	}

	if (include_writeoff == "include"):
		params["writeoff-categories"] = "on"
	elif (include_writeoff == "exclude"):
		params["exclude-writeoff-categories"] = "on"
	elif (include_writeoff == "writeoff-only"):
		params["only-writeoff-categories"] = "on"
		
	year = min_year
	page = 1
	attempt = 1


	try:

		while year <= max_year:

			params["year-from"] = year
			params["year-to"] = year
			params["page"] = page

			r = requests.get(url, params=params)
			if verbose:
				print("Year:     ", year)
				print("Page:     ", page)
				print("Response: ", r)

			try:
				j = r.json()
				s = BeautifulSoup(j["html"], features="html.parser")

				articles = s.find_all("article", attrs={"data-standout-type":""})

				# if no results or reached end of results...
				if len(articles) == 0 or r.url[r.url.find("page=")+5:] != str(page):
					if verbose:
						print("Found total", n_this_year_results, "results for year", year, "across", page-1, "pages")
						if year+1 <= max_year:
							print("Moving on to year", year + 1)
							print("---------------------------------")

					# Increment year and reset relevant variables
					year = year + 1
					page = 1
					attempt = 1
					n_this_year_results = 0
				else:
					for article in articles:
						car = {}
						car["name"] = article.find("h2", {"class": "listing-title"}).find("a").text.strip()					
						car["link"] = "https://www.autotrader.co.uk" + article.find("a", {"class": "tracking-standard-link"})["href"][: article.find("a", {"class": "tracking-standard-link"})["href"].find("?")]
						car["price"] = article.find("div", {"class": "vehicle-price"}).text.replace("RRP","").strip()

						key_specs_bs_list = article.find("ul", {"class": "listing-key-specs"}).find_all("li")
						
						for key_spec_bs_li in key_specs_bs_list:

							key_spec_bs = key_spec_bs_li.text

							if any(keyword in key_spec_bs for keyword in keywords["mileage"]):
								car["mileage"] = int(key_spec_bs[:key_spec_bs.find(" miles")].replace(",",""))
							elif any(keyword in key_spec_bs for keyword in keywords["BHP"]):
								car["BHP"] = int(key_spec_bs[:key_spec_bs.find("BHP")])
							elif any(keyword in key_spec_bs for keyword in keywords["transmission"]):
								car["transmission"] = key_spec_bs
							elif any(keyword in key_spec_bs for keyword in keywords["fuel"]):
								car["fuel"] = key_spec_bs
							elif any(keyword in key_spec_bs for keyword in keywords["owners"]):
								car["owners"] = int(key_spec_bs[:key_spec_bs.find(" owners")])
							elif any(keyword in key_spec_bs for keyword in keywords["body"]):
								car["body"] = key_spec_bs
							elif any(keyword in key_spec_bs for keyword in keywords["ULEZ"]):
								car["ULEZ"] = key_spec_bs
							elif any(keyword in key_spec_bs for keyword in keywords["year"]):
								car["year"] = key_spec_bs
							elif key_spec_bs[1] == "." and key_spec_bs[3] == "L":
								car["engine"] = key_spec_bs

						results.append(car)
						n_this_year_results = n_this_year_results + 1

					page = page + 1
					attempt = 1

					if verbose:
						print("Car count: ", len(results))
						print("---------------------------------")

			except KeyboardInterrupt:
				break

			except:
				traceback.print_exc()
				attempt = attempt + 1
				if attempt <= max_attempts_per_page:
					if verbose:
						print("Exception. Starting attempt #", attempt, "and keeping at page #", page)
				else:
					page = page + 1
					attempt = 1
					if verbose:
						print("Exception. All attempts exhausted for this page. Skipping to next page #", page)

	except KeyboardInterrupt:
		pass

	return results

### Output functions ###

def save_csv(results = [], filename = "scraper_output.csv"):
	csv_columns = ["name", "link", "price", "mileage", "BHP", "transmission", "fuel", "owners", "body", "ULEZ", "engine", "year"]

	with open(filename, "w", newline='') as f:
		writer = csv.DictWriter(f, fieldnames=csv_columns)
		writer.writeheader()
		for data in results:
			writer.writerow(data)

def save_json(results = [], filename = "scraper_output.json"):
	with open(filename, 'w') as f:
		json.dump(results, f, sort_keys=True, indent=4, separators=(',', ': '))