# %% #@ Libraries imports

import pandas as pd
import numpy as np
import requests
import json
from io import BytesIO
import copy
import zipfile as zp
import xlsxwriter
import xml.etree.ElementTree as ET
import uuid
from types import FunctionType
from .classes import Form
import docx as dcx
from docx.oxml.ns import nsdecls
from docx.oxml import parse_xml
from re import findall
from docx.oxml import OxmlElement
from docx.oxml.ns import qn
from docx.shared import RGBColor


# %% #@ Functions

def save_to_excel(data, filename="output.xlsx", column_width=25, include_index=False, row_colours={0: "#D8E4BC", 1: "#C5D9F1"}, row_bold=[0], row_wrap=[1], autofilter=True, freeze_panes=True):

    workbook = xlsxwriter.Workbook(filename)
    worksheet = workbook.add_worksheet()

    for i in range(len(data.columns)):
        worksheet.write(0, i, data.columns[i])

    for i in range(len(data)):
        for j in range(len(data.columns)):
            if pd.isna(data.iloc[i, j]):
                pass
            else:
                worksheet.write(i+1, j, data.iloc[i, j])

    worksheet.set_column(0, len(data.columns), width=column_width)

    for i in range(len(data)):
        a = {}
        if i in list(row_colours.keys()):
            a["bg_color"] = row_colours[i]
        if i in row_bold:
            a["bold"] = True
        if i in row_wrap:
            a["text_wrap"] = True
        if len(a) != 0:
            worksheet.set_row(i, cell_format=workbook.add_format(a))

    if autofilter:
        worksheet.autofilter(1, 0, len(data), len(data.columns)-1)

    if freeze_panes:
        worksheet.freeze_panes(2, 0)

    workbook.close()


class process_questionnaire():
    def __init__(self):
        self.attachments = None
        self.survey = None
        self.choices = None
        self.settings = None
        self.form_title = None
        self.form_version = None

    def get_data_from_files(self, form_filename, attachements_list_filenames):
        survey = pd.read_excel(form_filename, na_values=[
                               ' ', ''], keep_default_na=False, sheet_name="survey").dropna(how='all')
        self.survey = survey
        choices = pd.read_excel(form_filename, sheet_name="choices", na_values=[
                                ' ', ''], keep_default_na=False).dropna(how='all')
        self.choices = choices
        settings = pd.read_excel(form_filename, sheet_name="settings", na_values=[
            ' ', ''], keep_default_na=False).dropna(how='all')
        self.settings = settings
        self.form_title = self.settings['form_title'].iloc[0]
        self.form_version = self.settings['version'].iloc[0]
        attachments = {}
        for j in attachements_list_filenames:
            attachments[j] = pd.read_csv(j)
        self.attachments = attachments

    def get_data_from_odk_object(self, odk_object):
        self.survey = odk_object.survey
        self.choices = odk_object.choices
        self.settings = odk_object.settings
        self.form_title = self.settings['form_title'].iloc[0]
        self.form_version = self.settings['version'].iloc[0]
        self.attachments = odk_object.attachments

    def process(self):

        def set_repeat_table_header(row):
            """ set repeat table row on every new page
            """
            tr = row._tr
            trPr = tr.get_or_add_trPr()
            tblHeader = OxmlElement('w:tblHeader')
            tblHeader.set(qn('w:val'), "true")
            trPr.append(tblHeader)
            return row

        document = dcx.Document()
        section = document.sections[-1]
        new_width, new_height = section.page_height, section.page_width
        section.orientation = dcx.enum.section.WD_ORIENT.LANDSCAPE
        section.page_width = new_width
        section.page_height = new_height
        section.top_margin = dcx.shared.Cm(0.5)
        section.bottom_margin = dcx.shared.Cm(0.5)
        section.left_margin = dcx.shared.Cm(1)
        section.right_margin = dcx.shared.Cm(1)
        heading = document.add_heading(
            f"Form title = {self.form_title}\nForm version = {str(self.form_version)}\n\n")
        heading.alignment = 1
        table = document.add_table(rows=1, cols=6)
        table.style = "Table Grid"
        title_cells = table.rows[0].cells
        headings = ["Question code", "Question type",
                    "Question", "Hint", "Select Options", "Logic"]
        for i in range(len(headings)):
            paragraph = title_cells[i].paragraphs[0]
            run = paragraph.add_run(headings[i])
            run.bold = True

        def get_choices(choice):
            choice_names = list(
                self.choices["label::English (en)"].loc[self.choices["list_name"] == choice].map(str))
            choice_labels = list(
                self.choices["name"].loc[self.choices["list_name"] == choice].map(str))
            zipped = list(zip(choice_labels, choice_names))
            zipped = "\n".join([" = ".join(x) for x in zipped])
            return zipped

        def get_from_file(file):
            choice_list = list(self.attachments[file]["label"].map(str))
            return "\n".join(choice_list)

        def process_enclosing_variables(data):
            x = findall(r"\${.+?}", data)
            for i in x:
                data = data.replace(i, i[2:-1])
            return data

        def process_current_input(data):
            x = findall(r"\.", data)
            for i in x:
                data = data.replace(i, "input ")
            return data

        def process_string_only(column, index, cell_index):
            if not pd.isna(s[column].iloc[index]):
                row_cells[cell_index].text = process_enclosing_variables(
                    str(s[column].iloc[index]))
            else:
                row_cells[cell_index].text = ""

        def process_string_only_combined(cells, index):
            columns_names = ["Skip logic: ", "Default value: ", "Constrain value: ",
                             "Calculation: ", "Choice filter: ", "Repeat count: "]
            column_labels = ["relevant", "default", "constraint",
                             "calculation", "choice_filter", "repeat_count"]
            paragraph = cells[5].paragraphs[0]
            count = 0
            for j in range(len(column_labels)):
                if not pd.isna(self.survey[column_labels[j]].iloc[index]):
                    count += 1
            for j in range(len(column_labels)):
                if not pd.isna(self.survey[column_labels[j]].iloc[index]):
                    run = paragraph.add_run(columns_names[j])
                    run.italics = True
                    run.font.color.rgb = RGBColor(50, 0, 255)
                    if column_labels[j] == "constraint":
                        run = paragraph.add_run(
                            f"{process_current_input(process_enclosing_variables(str(self.survey[column_labels[j]].iloc[index])))}")
                    else:
                        run = paragraph.add_run(
                            f"{process_enclosing_variables(str(self.survey[column_labels[j]].iloc[index]))}")
                    count -= 1
                    if count != 0:
                        run = paragraph.add_run("\n")

        def cell_shading(index=None, index_counter=None, header_row=False):
            if index != None:
                sss = self.survey["type"].iloc[index].split(" ")[0]
                if sss == "begin_group":
                    shading_elm_0 = parse_xml(
                        r'<w:shd {} w:fill="BDBD51"/>'.format(nsdecls('w')))
                    table.rows[index+1 -
                               index_counter].cells[0]._tc.get_or_add_tcPr().append(shading_elm_0)
                    shading_elm_1 = parse_xml(
                        r'<w:shd {} w:fill="BDBD51"/>'.format(nsdecls('w')))
                    table.rows[index+1 -
                               index_counter].cells[1]._tc.get_or_add_tcPr().append(shading_elm_1)
                    shading_elm_2 = parse_xml(
                        r'<w:shd {} w:fill="BDBD51"/>'.format(nsdecls('w')))
                    table.rows[index+1 -
                               index_counter].cells[2]._tc.get_or_add_tcPr().append(shading_elm_2)
                    shading_elm_3 = parse_xml(
                        r'<w:shd {} w:fill="BDBD51"/>'.format(nsdecls('w')))
                    table.rows[index+1 -
                               index_counter].cells[3]._tc.get_or_add_tcPr().append(shading_elm_3)
                    shading_elm_4 = parse_xml(
                        r'<w:shd {} w:fill="BDBD51"/>'.format(nsdecls('w')))
                    table.rows[index+1 -
                               index_counter].cells[4]._tc.get_or_add_tcPr().append(shading_elm_4)
                    shading_elm_5 = parse_xml(
                        r'<w:shd {} w:fill="BDBD51"/>'.format(nsdecls('w')))
                    table.rows[index+1 -
                               index_counter].cells[5]._tc.get_or_add_tcPr().append(shading_elm_5)
                elif sss == "end_group":
                    shading_elm_0 = parse_xml(
                        r'<w:shd {} w:fill="E2E2B3"/>'.format(nsdecls('w')))
                    table.rows[index+1 -
                               index_counter].cells[0]._tc.get_or_add_tcPr().append(shading_elm_0)
                    shading_elm_1 = parse_xml(
                        r'<w:shd {} w:fill="E2E2B3"/>'.format(nsdecls('w')))
                    table.rows[index+1 -
                               index_counter].cells[1]._tc.get_or_add_tcPr().append(shading_elm_1)
                    shading_elm_2 = parse_xml(
                        r'<w:shd {} w:fill="E2E2B3"/>'.format(nsdecls('w')))
                    table.rows[index+1 -
                               index_counter].cells[2]._tc.get_or_add_tcPr().append(shading_elm_2)
                    shading_elm_3 = parse_xml(
                        r'<w:shd {} w:fill="E2E2B3"/>'.format(nsdecls('w')))
                    table.rows[index+1 -
                               index_counter].cells[3]._tc.get_or_add_tcPr().append(shading_elm_3)
                    shading_elm_4 = parse_xml(
                        r'<w:shd {} w:fill="E2E2B3"/>'.format(nsdecls('w')))
                    table.rows[index+1 -
                               index_counter].cells[4]._tc.get_or_add_tcPr().append(shading_elm_4)
                    shading_elm_5 = parse_xml(
                        r'<w:shd {} w:fill="E2E2B3"/>'.format(nsdecls('w')))
                    table.rows[index+1 -
                               index_counter].cells[5]._tc.get_or_add_tcPr().append(shading_elm_5)
                elif sss == "begin_repeat":
                    shading_elm_0 = parse_xml(
                        r'<w:shd {} w:fill="AAECD3"/>'.format(nsdecls('w')))
                    table.rows[index+1 -
                               index_counter].cells[0]._tc.get_or_add_tcPr().append(shading_elm_0)
                    shading_elm_1 = parse_xml(
                        r'<w:shd {} w:fill="AAECD3"/>'.format(nsdecls('w')))
                    table.rows[index+1 -
                               index_counter].cells[1]._tc.get_or_add_tcPr().append(shading_elm_1)
                    shading_elm_2 = parse_xml(
                        r'<w:shd {} w:fill="AAECD3"/>'.format(nsdecls('w')))
                    table.rows[index+1 -
                               index_counter].cells[2]._tc.get_or_add_tcPr().append(shading_elm_2)
                    shading_elm_3 = parse_xml(
                        r'<w:shd {} w:fill="AAECD3"/>'.format(nsdecls('w')))
                    table.rows[index+1 -
                               index_counter].cells[3]._tc.get_or_add_tcPr().append(shading_elm_3)
                    shading_elm_4 = parse_xml(
                        r'<w:shd {} w:fill="AAECD3"/>'.format(nsdecls('w')))
                    table.rows[index+1 -
                               index_counter].cells[4]._tc.get_or_add_tcPr().append(shading_elm_4)
                    shading_elm_5 = parse_xml(
                        r'<w:shd {} w:fill="AAECD3"/>'.format(nsdecls('w')))
                    table.rows[index+1 -
                               index_counter].cells[5]._tc.get_or_add_tcPr().append(shading_elm_5)
                elif sss == "end_repeat":
                    shading_elm_0 = parse_xml(
                        r'<w:shd {} w:fill="C5E8DA"/>'.format(nsdecls('w')))
                    table.rows[index+1 -
                               index_counter].cells[0]._tc.get_or_add_tcPr().append(shading_elm_0)
                    shading_elm_1 = parse_xml(
                        r'<w:shd {} w:fill="C5E8DA"/>'.format(nsdecls('w')))
                    table.rows[index+1 -
                               index_counter].cells[1]._tc.get_or_add_tcPr().append(shading_elm_1)
                    shading_elm_2 = parse_xml(
                        r'<w:shd {} w:fill="C5E8DA"/>'.format(nsdecls('w')))
                    table.rows[index+1 -
                               index_counter].cells[2]._tc.get_or_add_tcPr().append(shading_elm_2)
                    shading_elm_3 = parse_xml(
                        r'<w:shd {} w:fill="C5E8DA"/>'.format(nsdecls('w')))
                    table.rows[index+1 -
                               index_counter].cells[3]._tc.get_or_add_tcPr().append(shading_elm_3)
                    shading_elm_4 = parse_xml(
                        r'<w:shd {} w:fill="C5E8DA"/>'.format(nsdecls('w')))
                    table.rows[index+1 -
                               index_counter].cells[4]._tc.get_or_add_tcPr().append(shading_elm_4)
                    shading_elm_5 = parse_xml(
                        r'<w:shd {} w:fill="C5E8DA"/>'.format(nsdecls('w')))
                    table.rows[index+1 -
                               index_counter].cells[5]._tc.get_or_add_tcPr().append(shading_elm_5)
                elif (sss == "calculation"):
                    shading_elm_0 = parse_xml(
                        r'<w:shd {} w:fill="CECECE"/>'.format(nsdecls('w')))
                    table.rows[index+1 -
                               index_counter].cells[0]._tc.get_or_add_tcPr().append(shading_elm_0)
                    shading_elm_1 = parse_xml(
                        r'<w:shd {} w:fill="CECECE"/>'.format(nsdecls('w')))
                    table.rows[index+1 -
                               index_counter].cells[1]._tc.get_or_add_tcPr().append(shading_elm_1)
                    shading_elm_2 = parse_xml(
                        r'<w:shd {} w:fill="CECECE"/>'.format(nsdecls('w')))
                    table.rows[index+1 -
                               index_counter].cells[2]._tc.get_or_add_tcPr().append(shading_elm_2)
                    shading_elm_3 = parse_xml(
                        r'<w:shd {} w:fill="CECECE"/>'.format(nsdecls('w')))
                    table.rows[index+1 -
                               index_counter].cells[3]._tc.get_or_add_tcPr().append(shading_elm_3)
                    shading_elm_4 = parse_xml(
                        r'<w:shd {} w:fill="CECECE"/>'.format(nsdecls('w')))
                    table.rows[index+1 -
                               index_counter].cells[4]._tc.get_or_add_tcPr().append(shading_elm_4)
                    shading_elm_5 = parse_xml(
                        r'<w:shd {} w:fill="CECECE"/>'.format(nsdecls('w')))
                    table.rows[index+1 -
                               index_counter].cells[5]._tc.get_or_add_tcPr().append(shading_elm_5)
            if header_row:
                shading_elm_0 = parse_xml(
                    r'<w:shd {} w:fill="919191"/>'.format(nsdecls('w')))
                table.rows[0].cells[0]._tc.get_or_add_tcPr().append(
                    shading_elm_0)
                shading_elm_1 = parse_xml(
                    r'<w:shd {} w:fill="919191"/>'.format(nsdecls('w')))
                table.rows[0].cells[1]._tc.get_or_add_tcPr().append(
                    shading_elm_1)
                shading_elm_2 = parse_xml(
                    r'<w:shd {} w:fill="919191"/>'.format(nsdecls('w')))
                table.rows[0].cells[2]._tc.get_or_add_tcPr().append(
                    shading_elm_2)
                shading_elm_3 = parse_xml(
                    r'<w:shd {} w:fill="919191"/>'.format(nsdecls('w')))
                table.rows[0].cells[3]._tc.get_or_add_tcPr().append(
                    shading_elm_3)
                shading_elm_4 = parse_xml(
                    r'<w:shd {} w:fill="919191"/>'.format(nsdecls('w')))
                table.rows[0].cells[4]._tc.get_or_add_tcPr().append(
                    shading_elm_4)
                shading_elm_5 = parse_xml(
                    r'<w:shd {} w:fill="919191"/>'.format(nsdecls('w')))
                table.rows[0].cells[5]._tc.get_or_add_tcPr().append(
                    shading_elm_5)

        cell_shading(header_row=True)
        set_repeat_table_header(table.rows[0])
        index_counter = 0
        for i in range(len(self.survey)):
            s = self.survey
            c = self.choices
            if self.survey["relevant"].iloc[i] == "false()":
                index_counter += 1
            elif self.survey["type"].iloc[i].split(" ")[0] == "start":
                index_counter += 1
            elif self.survey["type"].iloc[i].split(" ")[0] == "end":
                index_counter += 1
            elif self.survey["type"].iloc[i].split(" ")[0] == "deviceid":
                index_counter += 1
            elif self.survey["type"].iloc[i].split(" ")[0] == "phonenumber":
                index_counter += 1
            else:
                row_cells = table.add_row().cells
                cell_shading(index=i, index_counter=index_counter)
                row_cells[0].text = s["name"].iloc[i] if not pd.isna(
                    s["name"].iloc[i]) else ""
                row_cells[1].text = s["type"].iloc[i].split(" ")[0]
                process_string_only("label::English (en)", i, 2)
                process_string_only("hint::English (en)", i, 3)
                if (s["type"].iloc[i].split(" ")[0] == "select_one") or (s["type"].iloc[i].split(" ")[0] == "select_multiple"):
                    row_cells[4].text = get_choices(
                        s["type"].iloc[i].split(" ")[1])
                elif (s["type"].iloc[i].split(" ")[0] == "select_one_from_file") or (s["type"].iloc[i].split(" ")[0] == "select_multiple_from_file"):
                    row_cells[4].text = get_from_file(
                        s["type"].iloc[i].split(" ")[1])
                else:
                    row_cells[4].text = ""
                process_string_only_combined(row_cells, i)

        document.save(f"{self.form_title}-{str(self.form_version)}.docx")
# %% #@ ODK Class

class ODK():

    def __init__(self, url):
        self.url = url
        self.form = None
        self.project = None
        self.survey = None
        self.choices = None
        self.settings = None
        self.attachments = None

    def connect(self, email, password):

        self.email = email
        self.password = password

        req = requests.post(self.url+'/v1/sessions', data=json.dumps(
            {"email": self.email, "password": self.password}), headers={'Content-Type': 'application/json'})

        self.token = req.json()["token"]
        self.headers = {'Authorization': 'Bearer '+self.token}

    def set_target(self, project_name, form_name):
        self.project_name = project_name
        self.form_name = form_name
        self.project = self.get_project()
        self.form = self.get_form()
        self.survey = self.get_survey()
        self.choiches = self.get_choices()
        self.settings = self.get_settings()
        self.attachments = self.get_attachments()

    def list_projects(self):
        req = requests.get(self.url+'/v1/projects', headers=self.headers)
        projects = [req.json()[i]["name"] for i in range(len(req.json()))]
        return projects

    def get_project(self):
        req = requests.get(self.url+'/v1/projects', headers=self.headers)
        project = [req.json()[i]["id"] for i in range(len(req.json()))
                   if req.json()[i]["name"] == self.project_name][0]

        return project

    def list_forms(self, project=None):
        req = requests.get(self.url+'/v1/projects', headers=self.headers)
        if project != None:
            project = [req.json()[i]["id"] for i in range(
                len(req.json())) if req.json()[i]["name"] == project][0]
        else:
            project = [req.json()[i]["id"] for i in range(
                len(req.json())) if req.json()[i]["name"] == self.project_name][0]
        req = requests.get(self.url+'/v1/projects/' +
                           str(project)+"/forms", headers=self.headers)
        forms = [req.json()[i]["name"] for i in range(len(req.json()))]
        return forms

    def get_form(self):

        req = requests.get(self.url+'/v1/projects/' +
                           str(self.get_project())+"/forms", headers=self.headers)
        form = [req.json()[i]["xmlFormId"] for i in range(len(req.json()))
                if req.json()[i]["name"] == self.form_name][0]

        return form

    def save_form(self, path="", save_file=True, xml=False):

        if xml:
            extension = '.xml'
        else:
            extension = '.xlsx'
            version = str(pd.read_excel(BytesIO(req),
                                        sheet_name="settings")["version"].iloc[0])

        req = requests.get(self.url+'/v1/projects/'+str(self.get_project()) +
                           "/forms/"+self.get_form()+extension, headers=self.headers).content

        if save_file:
            file = open(path+"form_v"+version+extension, "wb")
            file.write(req)
            file.close()
        else:
            return BytesIO(req)

    def get_submissions(self):

        req = (requests.get(self.url+'/v1/projects/' +
                            str(self.project)+"/forms/" +
                            self.form+"/submissions.csv?",
                            headers=self.headers))
        df = pd.read_csv(BytesIO(req.content))
        return df

    def get_survey(self):
        req = requests.get(self.url+'/v1/projects/'+str(self.project) +
                           "/forms/"+self.form+".xlsx", headers=self.headers)
        survey = pd.read_excel(BytesIO(req.content), na_values=[
                               ' ', ''], keep_default_na=False).dropna(how='all')
        self.survey = survey
        return survey

    def get_choices(self):
        req = requests.get(self.url+'/v1/projects/'+str(self.project) +
                           "/forms/"+self.form+".xlsx", headers=self.headers)
        choices = pd.read_excel(BytesIO(req.content), sheet_name="choices", na_values=[
                                ' ', ''], keep_default_na=False).dropna(how='all')
        self.choices = choices
        return choices

    def get_settings(self):
        req = requests.get(self.url+'/v1/projects/'+str(self.project) +
                           "/forms/"+self.form+".xlsx", headers=self.headers)
        settings = pd.read_excel(BytesIO(req.content), sheet_name="settings", na_values=[
                                 ' ', ''], keep_default_na=False).dropna(how='all')
        self.settings = settings
        return settings

    def get_repeats(self):

        req = (requests.get(self.url+'/v1/projects/' +
                            str(self.project)+"/forms/"+self.form +
                            "/submissions.csv.zip?attachments=false",
                            headers=self.headers))
        zipfile = zp.ZipFile(BytesIO(req.content))

        repeats = {}

        form_id = str(pd.read_excel(BytesIO(requests.get(self.url+'/v1/projects/'+str(self.project)+"/forms/"+self.form+".xlsx", headers=self.headers).content),
                                    sheet_name="settings")["form_id"].iloc[0])

        for j in self.survey["name"].loc[self.survey["type"] == "begin_repeat"]:
            repeats[form_id+"-" + j] = pd.read_csv(zipfile.open(
                form_id+"-" + j+".csv"), na_values=[' ', ''], keep_default_na=False).dropna(how='all')

        return repeats

    def get_attachments(self):

        req = (requests.get(self.url+'/v1/projects/' +
                            str(self.project)+"/forms/" +
                            self.form+"/attachments",
                            headers=self.headers))

        attachments = {}

        for j in req.json():
            attachments[j["name"]] = pd.read_csv(BytesIO((requests.get(self.url+'/v1/projects/' + str(self.project)+"/forms/"+self.form+"/attachments/"+j["name"], headers=self.headers)).content)) if j["name"].split(
                ".")[-1] == "csv" else BytesIO((requests.get(self.url+'/v1/projects/' + str(self.project)+"/forms/"+self.form+"/attachments/"+j["name"], headers=self.headers)).content)
        return attachments

    def get_media(self):

        req = requests.get(self.url+'/v1/projects/'+str(self.project) +
                           "/forms/"+self.form+".xlsx", headers=self.headers).content

        req = (requests.post(self.url+'/v1/projects/' +
                             str(self.project)+"/forms/" +
                             self.form+"/submissions.csv.zip?",
                             headers=self.headers))
        zipfile = zp.ZipFile(BytesIO(req.content))
        media = {}
        for name in zipfile.namelist():
            if (name.split('/')[0] == 'media') & (len(name) > 6):
                media[name.split('/')[-1]] = zipfile.read(name)
        return media

    def processing_submission(self, process_datetimes=False):

        df = self.get_submissions()

        def remove_tail(list_in):
            a = []
            for j in list_in:
                if j[-2:] == ".0":
                    a.append(j[:-2])
                else:
                    a.append(j)
            return a

        def select_one(select, value):
            x = self.survey["type"].loc[self.survey["name"] == select].iloc[0].split(" ")[
                1]
            y = self.choices["label::English (en)"].loc[self.choices["list_name"].map(lambda x: x.strip())
                                                        == x].loc[self.choices["name"] == value].iloc[0]
            return y

        def select_multiple(select, value):
            x = self.survey["type"].loc[self.survey["name"] == select].iloc[0].split(" ")[
                1]
            y = self.choices.loc[self.choices["list_name"].map(
                lambda x: x.strip()) == x]
            z = []
            for i in range(len(y)):
                if str(y["name"].iloc[i]) in remove_tail(list(str(value).split(" "))):
                    z.append(y["label::English (en)"].iloc[i])
            return " \n".join(z)

        def select_one_from_file(select, value):
            x = self.survey["type"].loc[self.survey["name"] == select].iloc[0].split(" ")[
                1]
            y = pd.read_csv(x)
            z = y["label"].loc[y["name"] == value].iloc[0]
            return z

        def select_multiple_from_file(select, value):
            x = self.survey["type"].loc[self.survey["name"] == select].iloc[0].split(" ")[
                1]
            y = pd.read_csv(x)
            z = []
            for i in range(len(y)):
                if str(y["name"].iloc[i]) in remove_tail(list(str(value).split(" "))):
                    z.append(y["label"].iloc[i])
            return " \n".join(z)

        func = {"select_one_from_file": select_one_from_file,
                "select_one": select_one, "select_multiple": select_multiple, "select_multiple_from_file": select_multiple_from_file}

        group_names = list(
            self.survey["name"].loc[self.survey["type"] == "begin_group"])
        group_names = sorted(group_names, key=len, reverse=True)

        column_names = sorted(list(set(self.survey["name"].loc[((self.survey["type"] != "begin_group") & (self.survey["type"] != "end_group") & (
            self.survey["type"] != "begin_repeat") & (self.survey["type"] != "end_repeat"))]).difference(set([np.nan, ""]))), key=len, reverse=True)

        df_columns = sorted(list(df.columns), key=len, reverse=True)

        for i in df_columns:
            for j in column_names:
                if i.endswith(j):
                    df = df.rename(columns={i: j})

        for i in df_columns:
            for j in group_names:
                if i.startswith(j):
                    df = df.rename(columns={i: i[len(j):]})

        for i in df.columns:
            # try:
            a = i
            b = self.survey["type"].loc[self.survey["name"] == a]
            if len(b) == 0:
                pass
            else:
                b = b.iloc[0].split(" ")[0]
                if b in list(func.keys()):
                    for j in range(len(df)):
                        if pd.isna(df[i].iloc[j]):
                            pass
                        else:
                            try:
                                df[i].iat[j] = func[b](a, df[i].iat[j])
                            except:
                                pass

        df = df.loc[df["ReviewState"] != "rejected"]

        if process_datetimes:
            df["SubmissionDate"] = pd.to_datetime(
                df["SubmissionDate"], format="%Y-%m-%dT%H:%M:%S.%fZ")
            df["start"] = pd.to_datetime(
                df["start"], format="%Y-%m-%dT%H:%M:%S.%f%z")

            for j in self.survey["name"].loc[self.survey["type"] == "datetime"]:
                try:
                    df[j] = pd.to_datetime(
                        df[j], format="%Y-%m-%dT%H:%M:%S.%f%z")
                except:
                    df[j] = pd.to_datetime(df[j], format="mixed")

            for j in self.survey["name"].loc[self.survey["type"] == "date"]:
                try:
                    df[j] = pd.to_datetime(df[j], format="%Y-%m-%d").dt.date
                except:
                    pass

            for j in self.survey["name"].loc[self.survey["type"] == "time"]:
                try:
                    df[j] = pd.to_datetime(
                        df[j], format="%H:%M:%S.%f%z").dt.time
                except:
                    pass

        return df

    def processing_repeats(self, data=None, process_datetimes=False):

        repeats = self.get_repeats()
        df = self.processing_submission() if type(data) == type(None) else data
        set_not_rejected = list(df["KEY"])

        def remove_tail(list_in):
            a = []
            for j in list_in:
                if j[-2:] == ".0":
                    a.append(j[:-2])
                else:
                    a.append(j)
            return a

        def select_one(select, value):
            x = self.survey["type"].loc[self.survey["name"] == select].iloc[0].split(" ")[
                1]
            y = self.choices["label::English (en)"].loc[self.choices["list_name"]
                                                        == x].loc[self.choices["name"] == value].iloc[0]
            return y

        def select_multiple(select, value):
            x = self.survey["type"].loc[self.survey["name"] == select].iloc[0].split(" ")[
                1]
            y = self.choices.loc[self.choices["list_name"] == x]
            z = []
            for i in range(len(y)):
                if str(y["name"].iloc[i]) in remove_tail(list(str(value).split(" "))):
                    z.append(y["label::English (en)"].iloc[i])
            return " \n".join(z)

        def select_one_from_file(select, value):
            x = self.survey["type"].loc[self.survey["name"] == select].iloc[0].split(" ")[
                1]
            y = pd.read_csv(x)
            z = y["label"].loc[y["name"] == value].iloc[0]
            return z

        func = {"select_one_from_file": select_one_from_file,
                "select_one": select_one, "select_multiple": select_multiple}

        group_names = list(
            self.survey["name"].loc[self.survey["type"] == "begin_group"])
        group_names = sorted(group_names, key=len, reverse=True)

        column_names = sorted(list(set(self.survey["name"].loc[((self.survey["type"] != "begin_group") & (self.survey["type"] != "end_group") & (
            self.survey["type"] != "begin_repeat") & (self.survey["type"] != "end_repeat"))]).difference(set([np.nan]))), key=len, reverse=True)

        for k in repeats.keys():
            for i in repeats[k].columns:
                # try:
                a = i
                b = self.survey["type"].loc[self.survey["name"] == a]
                if len(b) == 0:
                    pass
                else:
                    b = b.iloc[0].split(" ")[0]
                    if b in list(func.keys()):
                        for j in range(len(repeats[k])):
                            if pd.isna(repeats[k][i].iloc[j]):
                                pass
                            else:
                                try:
                                    repeats[k][i].iat[j] = func[b](
                                        a, repeats[k][i].iat[j])
                                except:
                                    pass

        for j in repeats.keys():

            repeats[j] = repeats[j].loc[[True if repeats[j]["PARENT_KEY"].iloc[i].split(
                "/")[0] in set_not_rejected else False for i in range(len(repeats[j]))]]

            if process_datetimes:

                for i in self.survey["name"].loc[self.survey["type"] == "datetime"]:
                    if i in repeats[j].columns:
                        try:
                            repeats[j][i] = pd.to_datetime(
                                repeats[j][i], format="%Y-%m-%dT%H:%M:%S.%f%z")
                        except:
                            repeats[j][i] = pd.to_datetime(
                                repeats[j][i], format="mixed")

                for i in self.survey["name"].loc[self.survey["type"] == "date"]:
                    if i in repeats[j].columns:
                        try:
                            repeats[j][i] = pd.to_datetime(
                                repeats[j][i], format="%Y-%m-%d").dt.date
                        except:
                            repeats[j][i] = pd.to_datetime(
                                repeats[j][i], format="mixed").dt.date

                for i in self.survey["name"].loc[self.survey["type"] == "time"]:
                    if i in repeats[j].columns:
                        try:
                            repeats[j][i] = pd.to_datetime(
                                repeats[j][i], format="%H:%M:%S.%f%z").dt.time
                        except:
                            repeats[j][i] = pd.to_datetime(
                                repeats[j][i], format="mixed").dt.time

        return repeats

    def process_all(self, variable='', time_variable='start', process_datetimes=False):

        submissions = self.processing_submission(
            process_datetimes=process_datetimes)
        survey = self.survey.dropna(how='all')
        choices = self.choices
        settings = self.settings
        repeats = self.processing_repeats(process_datetimes=process_datetimes)
        survey_name = self.form_name
        variable = variable
        time_variable = time_variable
        media = self.get_media()
        attachments = self.attachments

        return Form(submissions, survey, choices, settings, repeats, survey_name, variable, time_variable, media, attachments)

    def save_main(self, data=None, path=""):

        df = self.processing_submission() if type(data) == type(None) else data

        df_out = copy.deepcopy(df)

        for j in df_out.select_dtypes(include=['datetime64', 'datetimetz']).columns:
            df_out[j] = df_out[j].astype(str)
        if 'start' in df_out.columns:
            df_out['start'] = df_out['start'].astype(str)
        a = []
        for j in df.columns:
            if j in list(self.survey["name"]):
                x = self.survey["label::English (en)"].loc[self.survey["name"]
                                                           == j].iloc[0]
                a.append(x)
            else:
                a.append(np.nan)

        df_out.loc[-1] = a

        df_out.sort_index(inplace=True)

        save_to_excel(df_out, path+self.form_name+"_submissions.xlsx")

    def save_repeat(self, data=None, path=""):

        repeats = self.processing_repeats() if type(data) == type(None) else data

        for k in repeats.keys():

            rep_out = copy.deepcopy(repeats[k])

            for j in rep_out.select_dtypes(include=['datetime64', 'datetimetz']).columns:
                rep_out[j] = rep_out[j].astype(str)

            a = []
            for j in repeats[k].columns:
                if j in list(self.survey["name"]):
                    x = self.survey["label::English (en)"].loc[self.survey["name"]
                                                               == j].iloc[0]
                    a.append(x)
                else:
                    a.append(np.nan)

            rep_out.loc[-1] = a

            rep_out.sort_index(inplace=True)

            save_to_excel(rep_out, path+k+".xlsx")

    def save_data(self, path=""):

        req = requests.get(self.url+'/v1/projects/'+str(self.project) +
                           "/forms/"+self.form+".xlsx", headers=self.headers).content

        version = str(pd.read_excel(BytesIO(req),
                                    sheet_name="settings")["version"].iloc[0])
        req = (requests.post(self.url+'/v1/projects/' +
                             str(self.project)+"/forms/" +
                             self.form+"/submissions.csv.zip?",
                             headers=self.headers))

        file = open(path+self.form_name+"_v"+version+".zip", "wb")
        file.write(req.content)
        file.close()

    def listing_submissions(self):

        req = (requests.get(self.url+'/v1/projects/' +
                            str(self.project)+"/forms/" +
                            self.form+"/submissions",
                            headers=self.headers))
        return req.json()

    def get_submission_metadata(self, instance):

        req = (requests.get(self.url+'/v1/projects/' +
                            str(self.project)+"/forms/" +
                            self.form+"/submissions/"+instance,
                            headers=self.headers))
        return req.json()

    def get_submission_xml(self, instance):

        req = (requests.get(self.url+'/v1/projects/' +
                            str(self.project)+"/forms/" +
                            self.form+"/submissions/"+instance+".xml",
                            headers=self.headers))
        return req.content

    def put_submission(self, instance, data):

        req = (requests.put(url=self.url+'/v1/projects/' +
                            str(self.project)+"/forms/" +
                            self.form+"/submissions/"+instance, data=data,
                            headers=self.headers))
        return req

    def get_parent_tag(self, tag):

        n = self.survey.loc[self.survey['name'] == tag].index[0]
        begin_group = len(
            self.survey.iloc[:n].loc[self.survey['type'] == 'begin_group'])
        end_group = len(
            self.survey.iloc[:n].loc[self.survey['type'] == 'end_group'])
        begin_repeat = len(
            self.survey.iloc[:n].loc[self.survey['type'] == 'begin_repeat'])
        end_repeat = len(
            self.survey.iloc[:n].loc[self.survey['type'] == 'end_repeat'])

        if end_repeat < begin_repeat:
            return (self.survey['name'].iloc[:n].loc[self.survey['type'] == 'begin_repeat']).iloc[-1]
        elif end_group < begin_group:
            return (self.survey['name'].iloc[:n].loc[self.survey['type'] == 'begin_group']).iloc[-1]
        else:
            return None

    def return_element(self, tree, data: str):
        for elem in tree.iter():
            if elem.tag == data:
                return elem
            else:
                pass
        return None

    def modify_variable_xml(self, xml, variable: str, function):
        tree = ET.parse(BytesIO(xml))
        d = self.return_element(tree, variable)
        if d == None:
            print(f"{variable} is not in the xml")
            return xml
        else:
            try:
                k = d.text
                d.text = function(k)
                xml_out = BytesIO()
                tree.write(xml_out, encoding='utf-8')
                return xml_out.getvalue()
            except:
                print('an error occurred while processing for variable ', variable)
                return xml

    def update_xml(self, xml):

        tree = ET.parse(BytesIO(xml))
        root = tree.getroot()

        if tree.find('meta').find('deprecatedID') == None:
            old = tree.find('meta').find('instanceID').text
            tree.find('meta').find(
                'instanceID').text = 'uuid:'+str(uuid.uuid4())
            deprecated = ET.Element("deprecatedID")
            deprecated.text = old
            root.find('meta').append(deprecated)

        else:
            if len(tree.find('meta').find('deprecatedID').text) > 0:
                old = tree.find('meta').find('instanceID').text
                tree.find('meta').find(
                    'instanceID').text = 'uuid:'+str(uuid.uuid4())
                root.find('meta').find('deprecatedID').text = old
        xml_out = BytesIO()
        tree.write(xml_out, encoding='utf-8')
        return xml_out.getvalue()

    def change_submission(self, xml, id):
        self.put_submission(id, self.update_xml(xml))

    def drop_variable_xml(self, xml, variable: str, parent_tag=None):

        tree = ET.parse(BytesIO(xml))
        root = tree.getroot()
        for elem in tree.iter():
            if elem.tag == variable:
                if parent_tag == None:
                    root.remove(elem)
                else:
                    self.return_element(tree, parent_tag).remove(elem)
        xml_out = BytesIO()
        tree.write(xml_out, encoding='utf-8')
        return xml_out.getvalue()

    def add_variable_xml(self, xml, variable: str, parent_tag=None):

        tree = ET.parse(BytesIO(xml))
        root = tree.getroot()
        if type(self.return_element(tree, variable)) == type(None):
            if parent_tag == None:
                child = ET.SubElement(root, variable)
            else:
                child = ET.SubElement(
                    self.return_element(tree, parent_tag), variable)
            xml_out = BytesIO()
            tree.write(xml_out, encoding='utf-8')
            return xml_out.getvalue()
