import os
import json
import shutil
import datetime
import requests

from requests.auth import HTTPBasicAuth
from requests.exceptions import Timeout
from http.client import RemoteDisconnected


#-----------------------------------------------------------------------
def generate_date(start, end):
	start = datetime.datetime.strptime(start, "%Y/%m/%d")
	end = datetime.datetime.strptime(end, "%Y/%m/%d")
	for day in range(0, (end-start).days):
		date = start + datetime.timedelta(days=day)
		yield date


#-----------------------------------------------------------------------
def check_files(start, end):
	for date in generate_date(start, end):
		str_date = "{}".format(date.strftime("%Y-%m-%d"))
		filename = os.path.join("data/{}.json".format(str_date))
		if not os.path.exists(filename):
			print(str_date)


#-----------------------------------------------------------------------
def download_files(start, end):
	"""
	Download files between given date range.
	"""
	for date in generate_date(start, end):
		url = "https://dgarchive.caad.fkie.fraunhofer.de/d/{}".format(date.strftime("%Y/%m/%d"))
		print(url)

		try:
			r = requests.get(url, auth=HTTPBasicAuth("kolbik", "boorsitutrebleembedprod"), timeout=180)
		except Timeout:
			print("Timeout!")
			continue
		except RemoteDisconnected:
			print("Remote Disconnected!")
			continue

		print(r.status_code)
		if r.status_code != 200:
			print("WRONG STATUS CODE!")
			continue

		with open("{}.json".format(date.strftime("%Y-%m-%d")), "w") as out:
			out.write(json.dumps(r.json()))


#-----------------------------------------------------------------------
def process_jsons(start, end):
	for date in generate_date(start, end):
		with open("data/{}.json".format(date.strftime("%Y-%m-%d")), "r") as f:
			data = json.load(f)

			for key in data:
				with open("families/{}.txt".format(key), "a") as fam:
					for domain in data[key]:
						fam.write(domain + "\n")


#-----------------------------------------------------------------------
def unite_families():
	families = dict()
	for filename in os.listdir("families"):
		family = filename.split("_")[0]
		if family not in families:
			families[family] = [filename]
		else:
			families[family].append(filename)

	for family in families:
		with open("families_united/{}.txt".format(family), "w") as fw:
			for file in families[family]:
				with open("families/{}".format(file), "r") as fr:
					shutil.copyfileobj(fr, fw)


#-----------------------------------------------------------------------
#process_jsons("2017/01/01", "2020/02/01")
#unite_families()
#check_files("2017/01/01", "2020/02/01")
download_files("2017/01/01", "2020/02/01")
