import pandas as pd
import gzip
from os import listdir
from os.path import isfile, join
from tqdm import tqdm
import sys


def main(directory_path):
    only_files = [f for f in listdir(directory_path) if isfile(join(directory_path, f))]
    dataset_filenames = [name for name in only_files if name.startswith("c4-sk") and name.endswith(".json.gz")]

    for name in tqdm(dataset_filenames):
        try:
            with gzip.open(f'{directory_path}/{name}') as f:
                df = pd.read_json(f, lines=True)
                clean_text_name = name[:-len('.json.gz')]
                with open(f'{directory_path}/clean_text/{clean_text_name}.txt', 'w', encoding="utf-8") as o:
                    for text in df['text']:
                        o.write(text)
        except gzip.BadGzipFile:
            print(f"Bad gzip file: {name}")


if __name__ == '__main__':
    main(sys.argv[1])

