import csv
import json
import tldextract
from collections import defaultdict

def load_csv(filename):
    with open(filename, "r") as csvfile:
        reader = csv.DictReader(csvfile, delimiter=",")
        valid_words = []
        
        for row in reader:
            domain = tldextract.extract(row["domain"]).domain
            valid_words.append(domain)
    
    return valid_words

def load_words(filename):
    with open(filename, "r") as word_file:
        valid_words = list(word_file.read().split())
    return valid_words

def generate_ngrams(wordlist, n):
    ngrams_dict = defaultdict(int)

    for word in wordlist:
        word_length = len(word)
        if word_length >= n:
            for i in range(word_length - n + 1):
                ngrams_dict[word[i:i+n]] += 1

    ngrams_json = json.dumps(ngrams_dict, sort_keys=True, indent=4)
    output_filename = "{}grams.txt".format(str(n))
    with open(output_filename, "w") as of:
        of.write(ngrams_json)


wordlist = load_csv("tranco.csv")
generate_ngrams(wordlist, 5)