#  _  __
# | |/ /___ ___ _ __  ___ _ _ ®
# | ' </ -_) -_) '_ \/ -_) '_|
# |_|\_\___\___| .__/\___|_|
#              |_|
#
# Keeper Commander
# Copyright 2021 Keeper Security Inc.
# Contact: ops@keepersecurity.com
#

import copy
import datetime
import json
import logging
import re
from collections import Counter
from typing import Dict, Any

from .display import bcolors
from .params import KeeperParams
from .record import get_totp_code
from . import record_types, vault


class RecordV3:
    """Defines a user-friendly Keeper Record v3 for display purposes"""

    def __init__(self, record_uid='', folder='', title='', type='', fields=None, custom_fields=None, notes='',
                 revision='', data=''):
        self.record_uid = record_uid
        self.folder = folder
        self.title = title
        self.type = type
        self.fields = custom_fields or []
        self.custom_fields = custom_fields or []
        self.notes = notes
        self.revision = revision
        self.data = data

    @staticmethod
    def is_valid_record_type(record_type_json: str, rt_definition_json: str) -> dict:
        # validate record type (a.k.a. record v3) data
        # https://github.com/Keeper-Security/record-templates/tree/master/standard_templates
        rt = {}
        try:
            rt = json.loads(record_type_json)
        except ValueError:
            return {'is_valid': False, 'error': 'Invalid record type JSON'}
        except Exception:
            return {'is_valid': False, 'error': 'Invalid record type'}

        res = RecordV3.is_valid_record_type_definition(rt_definition_json)
        if not res.get('is_valid'):
            return res
        rtd = json.loads(rt_definition_json)

        # Implicit fields - always present on any record, no need to be specified in the template: title, custom, notes
        rtd_type = rtd.get('$id') or ''
        rt_type = rt.get('type') or ''
        rt_title = rt.get('title') or ''
        rt_notes = rt.get('notes') or ''
        rt_fields = rt.get('fields') or []
        rt_custom = rt.get('custom') or []

        if not isinstance(rt_type, str):
            return {'is_valid': False, 'error': 'Record type "type" should be string: ' + str(rt_type)}
        if not isinstance(rt_title, str):
            return {'is_valid': False, 'error': 'Record type "title" should be string: ' + str(rt_title)}
        if not isinstance(rt_notes, str):
            return {'is_valid': False, 'error': 'Record type "notes" should be string: ' + str(rt_notes)}
        if not isinstance(rt_fields, list):
            return {'is_valid': False, 'error': 'Record type "fields" should be array: ' + str(rt_fields)}
        if not isinstance(rt_custom, list):
            return {'is_valid': False, 'error': 'Record type "custom" should be array: ' + str(rt_custom)}

        if not rt_type or not rt_type.strip():
            return {'is_valid': False, 'error': 'Missing record type type'}
        if rt_type.strip() != rtd_type.strip():
            return {'is_valid': False,
                    'error': 'Type mistmatch - RT "type" should match "$id" in RT definition: ' + rt_type + " != " + rtd_type}

        if not rt_title or not rt_title.strip():
            return {'is_valid': False, 'error': 'Missing record type title'}

        # Top level RT attributes - unknown attribute(s) generate error
        # All known attribute(s) from RT definition that don't belong to RT data also generate error
        tlod = [x for x in rtd]
        tlor = [x for x in rt]
        rtdef_only = ('$id', 'categories', 'description', 'label')  # these are in RT definitions only
        ilist = [x for x in tlor if x in rtdef_only]
        if ilist:
            return {'is_valid': False,
                    'error': 'Record has atributes that should be present in record type definitions only: ' + str(
                        ilist)}

        # ulist = [x for x in tlor if x not in ('type', 'title', 'notes', 'fields', 'custom')]
        ulist = [x for x in tlor if x not in tlod + [*RecordV3.record_fields.keys()]]
        if ulist:
            return {'is_valid': False, 'error': 'Unknown record type attribute(s): ' + str(ulist)}

        # Allow only valid/known field types - field types are case sensitive?
        badf = [x for x in rt_fields if not RecordV3.field_types.__contains__(x.get('type'))]
        if badf:
            return {'is_valid': False, 'error': 'Unknown field types in "fields": ' + str(badf)}
        badf = [x for x in rt_custom if not RecordV3.field_types.__contains__(x.get('type'))]
        if badf:
            return {'is_valid': False, 'error': 'Unknown field types in "custom": ' + str(badf)}

        # only one FT of type password allowed in record v3
        pwdf = [x for x in rt_fields if x.get('type') == 'password']
        pwdc = [x for x in rt_custom if x.get('type') == 'password']
        pwds = pwdf + pwdc
        if len(pwds) > 1:
            return {'is_valid': False, 'error': 'Error: Only one password allowed per record! ' + str(pwds)}
        elif len(pwds) == 1:
            rtdp = next((True for x in (rtd.get('fields') or []) if '$ref' in x and x.get('$ref') == 'password'), False)
            if rtdp:
                if pwdc: return {'is_valid': False,
                                 'error': 'Password must be in fields[] section as defined by record type! ' + str(
                                     pwds)}
            else:
                if pwdf: return {'is_valid': False,
                                 'error': 'Password must be in custom[] section - this record type does not allow password in fields[]! ' + str(
                                     pwds)}

        # All fields in fields[] must be in record type definition
        rtdf = [x.get('$ref') for x in (rtd.get('fields') or []) if '$ref' in x]
        badf = [x for x in rt_fields if x.get('type') not in rtdf]
        if badf:
            return {'is_valid': False,
                    'error': 'This record type doesn\'t allow these in fields[] (move to custom): ' + str(badf)}

        # fileRef use upload-attachment/delete-attachment commands
    # remove from validation to allow for cross referencing same fileRef from multiple records v3
        # refs = [x for x in rt_fields + rt_custom if x.get('type') == 'fileRef' and x.get('value')]
        # if refs:
        #   return { 'is_valid': False, 'error': 'File reference manipulations are disabled here. Use upload-attachment/delete-attachment commands instead. ' + str(refs) }

        # fields[] must contain all 'required' fields (and required value parts) - custom[] is not in RT definition
        reqd = [x.get('$ref') for x in (rtd.get('fields') or []) if '$ref' in x and 'required' in x]
        reqf = [x for x in rt_fields if x.get('type') in reqd]
        regf = [x for x in rt_fields if x.get('type') not in reqd] + rt_custom

        # all fields required by RT definition must be present
        miss = set(reqd) - set([x.get('type') for x in reqf])
        if miss:
            return {'is_valid': False, 'error': 'Missing required fields: ' + str(miss)}

        # validate field values
        fver = []
        for fld in reqf:
            err = RecordV3.is_valid_field_data(fld, True)
            fver.extend(err)
        if fver:
            return {'is_valid': False, 'error': 'Error(s) validating required fields: ' + str(fver)}

        for fld in regf:
            err = RecordV3.is_valid_field_data(fld, False)
            fver.extend(err)
        if fver:
            return {'is_valid': False, 'error': 'Error(s) validating fields: ' + str(fver)}

        return {'is_valid': True, 'error': ''}

    @staticmethod
    def is_valid_record_type_definition(record_type_definition_json: str) -> dict:
        # validate record type (a.k.a. record v3) definition
        # https://github.com/Keeper-Security/record-templates/tree/master/standard_templates

        rt = {}
        try:
            rt = json.loads(record_type_definition_json)
        except ValueError:
            return {'is_valid': False, 'error': 'Invalid record type JSON'}
        except Exception:
            return {'is_valid': False, 'error': 'Invalid record type definition'}

        rtid = rt.get('$id') or ''
        if not rtid or not rtid.strip():
            return {'is_valid': False, 'error': 'Missing record type name'}

        # Min RT definition: {"$id": "Name"} - allows only implicit (custom) fields
        # Max RT definition: {"$id": "Name", "categories": ["note"], "description": "Description", "fields":[]}
        # Implicit fields - always present on any record, no need to be specified in the template: title, custom, notes
        implicit_field_names = [x for x in RecordV3.record_implicit_fields]
        implicit_fields = [r for r in rt if r in implicit_field_names]
        if implicit_fields:
            return {'is_valid': False,
                    'error': 'Implicit fields not allowed in record type definition: ' + str(implicit_fields)}

        rt_attributes = ('$id', 'categories', 'description', 'fields')
        bada = [r for r in rt if r not in rt_attributes and r not in implicit_field_names]
        if bada:
            logging.debug(f'Unknown attributes in record type definition: {bada}')

        # Allow only valid/known field types - field types are case sensitive?
        flds = rt.get('fields') or []
        # Record type definitions without fields are OK? They still have title and custom[]
        # if not flds:
        #   return { 'is_valid': False, 'error': 'Missing fields list' }

        badf = [x for x in flds if not x.get('$ref')]
        if badf:
            return {'is_valid': False, 'error': 'Missing field type reference (ex. {"$ref": "login"}): ' + str(badf)}

        badf = [x for x in flds if not RecordV3.field_types.__contains__(x.get('$ref'))]
        if badf:
            return {'is_valid': False, 'error': 'Unknown field types: ' + str(badf)}

        known_ft_atributes = {'$ref', 'label', 'required', 'privacyScreen', 'enforceGeneration', 'complexity'}
        unknown_ft_atributes = [x for x in flds if not set(x.keys()).issubset(known_ft_atributes)]
        if unknown_ft_atributes:
            return {'is_valid': False, 'error': 'Unknown field atributes: ' + str(unknown_ft_atributes)}

        badf = [x for x in flds if not RecordV3.is_valid_field_type_ref(json.dumps(x))]
        if badf:
            return {'is_valid': False, 'error': 'Invalid field types: ' + str(badf)}

        return {'is_valid': True, 'error': ''}

    # Implicit fields - The following fields are always present on any record and do not need to be specified in the template:
    record_implicit_fields = {
        'title': '',  # string
        'custom': [],  # Array of Field Data objects
        'notes': ''  # string
    }

    record_fields = {
        'title': '',  # string - Record Title
        'type': '',  # string - Record Type (either one of the standard types or a custom type)
        'fields': [],  # Array of Field Data objects
        'custom': [],  # Array of Field Data objects
        'notes': ''  # string
    }

    # Fields are client-side defined. Fields are defined with a Field Type. All Field Types are fixed.
    # Meaning, although we may implement new field types, the client cannot arbitrarily add field types,
    # nor display/edit field types that are not pre programmed into the client.
    # Once the client is updated to understand the new field type, the data may then be displayed and edited.

    # TODO: translatable labels
    # NB! Allow only one login/password field per v3 record. Allow in custom[] only if RT definition doesn't have them
    # Standard RT definitions: always required - type, title
    record_types = {
        'login': {'label': 'Login'},
        # def.  {"$id":"login","categories":["login"],"description":"Login template","fields":[{"$ref":"login"},{"$ref":"password"},{"$ref":"url"},{"$ref":"securityQuestion"},{"$ref":"fileRef"},{"$ref":"oneTimeCode"}]}
        # ex.   {"title":"Title","type":"login","fields":[{"type":"login","value":[]},{"type":"password","value":[]},{"type":"url","value":[]},{"type":"securityQuestion","value":[]},{"type":"fileRef","value":[]},{"type":"oneTimeCode","value":[]}],"custom":[]}

        'bankAccount': {'label': 'Bank Account'},  # { "$ref": "bankAccount", "required": true }
        # def.  {"$id":"bankAccount","description":"Bank account template","fields":[{"$ref":"bankAccount","required":true},{"$ref":"name"},{"$ref":"login"},{"$ref":"password"},{"$ref":"url"},{"$ref":"cardRef"},{"$ref":"fileRef"},{"$ref":"oneTimeCode"}]}
        # ex.   {"title":"Title","type":"bankAccount","fields":[{"type":"bankAccount","value":[{"accountType":"","routingNumber":"","accountNumber":"1234","otherType":""}],"required":true},{"type":"name","value":[{"first":"John","last":"Doe"}]},{"type":"login","value":[]},{"type":"password","value":[]},{"type":"url","value":[]},{"type":"cardRef","value":[]},{"type":"fileRef","value":[]},{"type":"oneTimeCode","value":[]}],"custom":[]}

        'address': {'label': 'Address'},
        # def.  {"$id":"address","categories":["address"],"description":"Address template","fields":[{"$ref":"address"},{"$ref":"fileRef"}]}
        # ex.   {"title":"Title","type":"address","fields":[{"type":"address","value":[]},{"type":"fileRef","value":[]}],"custom":[]}

        'bankCard': {'label': 'Payment Card'},
        # def.  {"$id":"bankCard","categories":["payment"],"description":"Bank card template","fields":[{"$ref":"paymentCard"},{"$ref":"text","label":"cardholderName"},{"$ref":"pinCode"},{"$ref":"addressRef"},{"$ref":"fileRef"}]}
        # ex.   {"title":"Title","type":"bankCard","fields":[{"type":"paymentCard","value":[]},{"type":"text","label":"Cardholder Name","value":[]},{"type":"pinCode","value":[]},{"type":"addressRef","value":[]},{"type":"fileRef","value":[]}],"custom":[]}

        'birthCertificate': {'label': 'Birth Certificate'},
        # def.  {"$id":"birthCertificate","categories":["ids"],"description":"Birth certificate template","fields":[{"$ref":"name"},{"$ref":"birthDate"},{"$ref":"fileRef"}]}
        # ex.   {"title":"Title","type":"birthCertificate","fields":[{"type":"name","value":[]},{"type":"birthDate","value":[]},{"type":"fileRef","value":[]}],"custom":[]}

        'contact': {'label': 'Contact'},  # { "$ref": "name", "required": true }
        # def.  {"$id":"contact","categories":["address"],"description":"Contact template","fields":[{"$ref":"name","required":true},{"$ref":"text","label":"company"},{"$ref":"email"},{"$ref":"phone"},{"$ref":"addressRef"},{"$ref":"fileRef"}]}
        # ex.   {"title":"Title","type":"contact","fields":[{"type":"name","value":[{"first":"John","last":"Doe"}],"required":true},{"type":"text","label":"Company","value":[]},{"type":"email","value":[]},{"type":"phone","value":[]},{"type":"addressRef","value":[]},{"type":"fileRef","value":[]}],"custom":[]}

        'driverLicense': {'label': 'Driver\'s License'},
        # def.  {"$id":"driverLicense","categories":["ids"],"description":"Driver license template","fields":[{"$ref":"accountNumber","label":"dlNumber"},{"$ref":"name"},{"$ref":"birthDate"},{"$ref":"addressRef"},{"$ref":"expirationDate"},{"$ref":"fileRef"}]}
        # ex.   {"title":"Title","type":"driverLicense","fields":[{"type":"accountNumber","label":"Driver's License Number","value":[]},{"type":"name","value":[]},{"type":"birthDate","value":[]},{"type":"addressRef","value":[]},{"type":"expirationDate","value":[]},{"type":"fileRef","value":[]}],"custom":[]}

        'encryptedNotes': {'label': 'Secure Note'},
        # def.  {"$id":"encryptedNotes","categories":["note"],"description":"Encrypted note template","fields":[{"$ref":"note"},{"$ref":"date"},{"$ref":"fileRef"}]}
        # ex.   {"title":"Title","type":"encryptedNotes","fields":[{"type":"note","value":[]},{"type":"date","value":[]},{"type":"fileRef","value":[]}],"custom":[]}

        'file': {'label': 'File Attachment'},
        # def.  {"$id":"file","categories":["file"],"description":"File template","fields":[{"$ref":"fileRef"}]}
        # ex.   {"title":"Title","type":"file","fields":[{"type":"fileRef","value":[]}],"custom":[]}

        'healthInsurance': {'label': 'Health Insurance'},
        # def.  {"$id":"healthInsurance","categories":["ids"],"description":"Health insurance template","fields":[{"$ref":"accountNumber"},{"$ref":"name","label":"insuredsName"},{"$ref":"login"},{"$ref":"password"},{"$ref":"url"},{"$ref":"fileRef"},{"$ref":"securityQuestion"}]}
        # ex.   {"title":"Title","type":"healthInsurance","fields":[{"type":"accountNumber","value":[]},{"type":"name","label":"Insured's Name","value":[]},{"type":"login","value":[]},{"type":"password","value":[]},{"type":"url","value":[]},{"type":"fileRef","value":[]},{"type":"securityQuestion","value":[]}],"custom":[]}

        'membership': {'label': 'Membership'},
        # def.  {"$id":"membership","categories":["ids"],"description":"Membership template","fields":[{"$ref":"accountNumber"},{"$ref":"name"},{"$ref":"password"},{"$ref":"fileRef"},{"$ref":"securityQuestion"}]}
        # ex.   {"title":"Title","type":"membership","fields":[{"type":"accountNumber","value":[]},{"type":"name","value":[]},{"type":"password","value":[]},{"type":"fileRef","value":[]},{"type":"securityQuestion","value":[]}],"custom":[]}

        'passport': {'label': 'Passport'},
        # def.  {"$id":"passport","categories":["ids"],"description":"Passport template","fields":[{"$ref":"accountNumber","label":"passportNumber"},{"$ref":"name"},{"$ref":"birthDate"},{"$ref":"addressRef"},{"$ref":"expirationDate"},{"$ref":"date","label":"dateIssued"},{"$ref":"password"},{"$ref":"fileRef"}]}
        # ex.   {"title":"Title","type":"passport","fields":[{"type":"accountNumber","label":"Passport Number","value":[]},{"type":"name","value":[]},{"type":"birthDate","value":[]},{"type":"addressRef","value":[]},{"type":"expirationDate","value":[]},{"type":"date","label":"Date Issued","value":[]},{"type":"password","value":[]},{"type":"fileRef","value":[]}],"custom":[]}

        'photo': {'label': 'Photo'},
        # def.  {"$id":"photo","categories":["file"],"description":"Photo template","fields":[{"$ref":"fileRef"}]}
        # ex.   {"title":"Title","type":"photo","fields":[{"type":"fileRef","value":[]}],"custom":[]}

        'serverCredentials': {'label': 'Server'},
        # def.  {"$id":"serverCredentials","categories":["login"],"description":"Server credentials template","fields":[{"$ref":"host"},{"$ref":"login"},{"$ref":"password"},{"$ref":"fileRef"}]}
        # ex.   {"title":"Title","type":"serverCredentials","fields":[{"type":"host","value":[]},{"type":"login","value":[]},{"type":"password","value":[]},{"type":"fileRef","value":[]}],"custom":[]}

        'softwareLicense': {'label': 'Software License'},
        # def.  {"$id":"softwareLicense","categories":["note"],"description":"Software license template","fields":[{"$ref":"licenseNumber"},{"$ref":"expirationDate"},{"$ref":"date","label":"dateActive"},{"$ref":"fileRef"},{"$ref":"securityQuestion"}]}
        # ex.   {"title":"Title","type":"softwareLicense","fields":[{"type":"licenseNumber","value":[]},{"type":"expirationDate","value":[]},{"type":"date","label":"Date Active","value":[]},{"type":"fileRef","value":[]},{"type":"securityQuestion","value":[]}],"custom":[]}

        'ssnCard': {'label': 'Identity Card'},
        # def.  {"$id":"ssnCard","categories":["ids"],"description":"Identity card template","fields":[{"$ref":"accountNumber","label":"identityNumber"},{"$ref":"name"},{"$ref":"fileRef"}]}
        # ex.   {"title":"Title","type":"ssnCard","fields":[{"type":"accountNumber","label":"Identity Number","value":[]},{"type":"name","value":[]},{"type":"fileRef","value":[]}],"custom":[]}

        'general': {'label': 'Legacy Record'},
        # def.  {"$id":"general","categories":["login"],"description":"Legacy template","fields":[{"$ref":"login"},{"$ref":"password"},{"$ref":"url"},{"$ref":"oneTimeCode"},{"$ref":"fileRef"}]}
        # ex.   {"title":"Title","type":"general","fields":[{"type":"login","value":[]},{"type":"password","value":[]},{"type":"url","value":[]},{"type":"oneTimeCode","value":[]},{"type":"fileRef","value":[]}],"custom":[],"notes":""}

        'sshKeys': {'label': 'SSH Key'},
        # def.  {"$id":"sshKeys","categories":["login"],"description":"SSH key template","fields":[{"$ref":"login"},{"$ref":"keyPair"},{"$ref":"text","label":"passphrase"},{"$ref":"host"},{"$ref":"fileRef"},{"$ref":"securityQuestion"}]}
        # ex.   {"title":"Title","type":"sshKeys","fields":[{"type":"login","value":[]},{"type":"keyPair","value":[]},{"type":"text","label":"Passphrase","value":[]},{"type":"host","value":[]},{"type":"fileRef","value":[]},{"type":"securityQuestion","value":[]}],"custom":[]}

        'databaseCredentials': {'label': 'Database'}
        # def.  {"$id":"databaseCredentials","categories":["login"],"description":"Database credentials template","fields":[{"$ref":"text","label":"type"},{"$ref":"host"},{"$ref":"login"},{"$ref":"password"},{"$ref":"fileRef"}]}
        # ex.   {"title":"Title","type":"databaseCredentials","fields":[{"type":"text","label":"Type","value":[]},{"type":"host","value":[]},{"type":"login","value":[]},{"type":"password","value":[]},{"type":"fileRef","value":[]}],"custom":[]}
    }

    # A field definition consists from:
    #   $id - ex. 'title', used to reference the field definitions and to translate the field labels
    #   type|$type - used to tell the UI how to render and edit the field
    #   lookup - optional flag: lookup values are shared based on the field reference.
    #     ex. all 'login' fields will share the same set of lookup values but will not see the lookup values from 'company' field
    #   multiple - optional flag: If specified, the UI should allow populating multiple instances of the field on the record
    #   label - optional modifier: If specified, the UI display the label instead of the translated value
    #   default - optional modifier: Defines the default value for the field
    #   readonly -  flag: restrict editable field to be readonly or relax readonly field to be editable
    #   required - optional flag: Can be used to make the field required
    #   hidden - optional flag. Can be used to hide a field from the base template

    # https://github.com/Keeper-Security/record-templates/blob/master/fields.json
    # NB! All field_types must have corresponding field_values
    field_types = {
        # 2021-05-06 FT 'text' added for compatibility with web vault
        'text': {
            '$id': 'text',
            'type': 'text'
        },
        # 2021-06-24 One RT added new FT 'secret' - type is yet unknown
        'secret': {
            '$id': 'secret',
            'type': 'text'
        },
        'title': {
            '$id': 'title',
            'type': 'text'
        },
        'login': {
            '$id': 'login',
            'type': 'login',
            'lookup': 'login'
        },
        'password': {
            '$id': 'password',
            'type': 'password'
        },
        'name': {
            '$id': 'name',
            'type': 'name',
            'lookup': 'name'
        },
        'company': {
            '$id': 'company',
            'type': 'text',
            'lookup': 'company'
        },
        'phone': {
            '$id': 'phone',
            'type': 'phone',
            'multiple': 'optional',
            'lookup': 'phone'
        },
        'email': {
            '$id': 'email',
            'type': 'email',
            'multiple': 'optional',
            'lookup': 'email'
        },
        'address': {
            '$id': 'address',
            'type': 'address',
            'label': 'Street Address'
        },
        'addressRef': {
            '$id': 'addressRef',
            'type': 'addressRef',
            'lookup': 'addressRef'
        },
        # 'cardNumber': {
        #   '$id': 'cardNumber',
        #   'type': 'cardNumber'  # undefined - probably replaced by paymentCard
        # },
        'date': {
            '$id': 'date',
            'type': 'date'
        },
        # 2021-06-18 One RT added new FT expirationDate probably just a date value
        'expirationDate': {
            '$id': 'expirationDate',
            'type': 'date'
        },
        'birthDate': {
            '$id': 'birthDate',
            'type': 'date'
        },
        'paymentCard': {
            '$id': 'paymentCard',
            'type': 'paymentCard'
        },
        'accountNumber': {
            '$id': 'accountNumber',
            'type': 'text',
            'lookup': 'accountNumber',
            'label': 'Account Number'
        },
        'groupNumber': {
            '$id': 'groupNumber',
            'type': 'text'
        },
        'bankAccount': {
            '$id': 'bankAccount',
            'type': 'bankAccount',
            'lookup': 'accountNumber'
        },
        'cardRef': {
            '$id': 'cardRef',
            'type': 'cardRef',
            'lookup': 'bankCard',
            'multiple': 'default'
        },
        'note': {
            '$id': 'note',
            'type': 'multiline'
        },
        'url': {
            '$id': 'url',
            'type': 'url',
            'multiple': 'optional'
        },
        # 2021-05-06 Unused photo, file - removed for compatibility with web vault
        # 'photo': {
        #   '$id': 'photo',
        #   'type': 'file'
        # },
        # 'file': {
        #   '$id': 'file',
        #   'type': 'file'
        # },
        'fileRef': {
            '$id': 'fileRef',
            'type': 'fileRef',
            'multiple': 'default'
        },
        'host': {
            '$id': 'host',
            'type': 'host',
            'lookup': 'host',
            'multiple': 'optional'
        },
        'securityQuestion': {
            '$id': 'securityQuestion',
            'type': 'securityQuestion',
            'multiple': 'default'
        },
        'pinCode': {
            '$id': 'pinCode',
            'type': 'secret'
        },
        'oneTimeCode': {
            '$id': 'oneTimeCode',
            'type': 'otp'
        },
        'keyPair': {
            '$id': 'keyPair',
            'type': 'privateKey'
        },
        'licenseNumber': {
            '$id': 'licenseNumber',
            'type': 'multiline'
        },
        'multiline': {
            '$id': 'multiline',
            'type': 'multiline'
        },
        'json': {
            '$id': 'json',
            'type': 'json'
        },
        'passkey': {
            '$id': 'passkey',
            'type': 'passkey'
        },
        # 'custom: {
        #   '$id': 'custom',
        #   'type': 'custom'
        # }
    }

    # https://github.com/Keeper-Security/record-templates/blob/master/field-types.json
    # Record labels override default field type labels
    field_values = {
        'text': {
            'type': 'text',
            'value_description': 'plain text',
            'value': ''  # string
        },
        'url': {
            'type': 'url',
            'value_description': 'url string, can be clicked',
            'value': ''  # string
        },
        'multiline': {
            'type': 'multiline',
            'value_description': 'multiline text',
            'value': ''  # string
        },
        'json': {
            'type': 'json',
            'value_description': 'json text, only validated data persisted',
            'value': ''  # string
        },
        # 2021-05-06 Unused file type - removed for compatibility with web vault
        # 'file': {
        #   'type': 'file',
        #   'value_description': 'large binary object, picture or other file',
        #   'value': '' # ???
        # },
        'fileRef': {
            'type': 'fileRef',
            'value_description': 'reference to the file field on another record',
            'value': ''  # string (record v4 UID)
        },
        'email': {
            'type': 'email',
            'value_description': 'valid email address plus tag',
            'value': ''  # string
        },
        'host': {
            'type': 'host',
            'value_description': 'multiple fields to capture host information',
            'value': {  # object
                'hostName': '',  # string
                'port': ''  # string
            }
        },
        'phone': {
            'type': 'phone',
            'value_description': 'numbers and symbols only plus tag',
            'value': {  # object
                'region': '',  # string
                'number': '',  # string
                'ext': '',  # string
                'type': ('', 'Mobile', 'Home', 'Work')
            }
        },
        'name': {
            'type': 'name',
            'value_description': 'multiple fields to capture name',
            'value': {  # object
                'first': '',  # string
                'middle': '',  # string
                'last': ''  # string
            },
            'required': ['first', 'last']
            # required parts of field value - enforced only when RT specify that field is required
            # 2021-04-01 use reference implementation (webVault - above) != from documentation (Record+Format+V3+draft - below)
            # 'value': {          # object
            #   'firstName': '',  # string
            #   'middleName': '', # string
            #   'lastName': ''    # string
            # },
            # 'required': ['firstName', 'lastName']
        },
        'address': {
            'type': 'address',
            'value_description': 'multiple fields to capture address',
            'value': {  # object
                'street1': '',  # string
                'street2': '',  # string
                'city': '',  # string
                'state': '',  # string
                'zip': '',  # string
                'country': ''  # string
            }
        },
        'addressRef': {
            'type': 'addressRef',
            'value_description': 'reference to the address field on another record',
            'value': ''  # string
        },
        'cardRef': {
            'type': 'cardRef',
            'value_description': 'reference to the bankCard field on another record',
            'value': ''  # string (record UID)
        },
        'secret': {
            'type': 'secret',
            'value_description': 'the field value is masked',
            'value': ''  # string
        },
        'login': {
            'type': 'login',
            'value_description': 'Login field, detected as the website login for browser extension or KFFA.',
            'value': ''  # string?
        },
        'password': {
            'type': 'password',
            'value_description': 'Field value is masked and allows for generation. Also complexity enforcements.',
            'value': ''  # string
        },
        'securityQuestion': {
            'type': 'securityQuestion',
            'value_description': 'Security Question and Answer',
            'value': {  # object
                'question': '',  # string
                'answer': ''  # string
            }
        },
        'otp': {
            'type': 'otp',
            'value_description': 'captures the seed, displays QR code',
            'value': ''  # string
        },
        'paymentCard': {
            'type': 'paymentCard',
            'value_description': 'Field consisting of validated card number, expiration date and security code.',
            'value': {  # object
                'cardNumber': '',  # string
                'cardExpirationDate': '',  # string
                'cardSecurityCode': ''  # string
            }
        },
        'date': {
            'type': 'date',
            'value_description': 'calendar date with validation, stored as unix milliseconds',
            'value': 0  # number (long)
        },
        'bankAccount': {
            'type': 'bankAccount',
            'value_description': 'bank account information',
            'value': {  # object
                'accountType': ('', 'Checking', 'Savings', 'Other'),
                'otherType': '',  # string
                'routingNumber': '',  # string
                'accountNumber': ''  # string (required for RT bankAccount)
            },
            'required': ['accountNumber']
            # required parts of field value - enforced only when RT specify that field is required
        },
        'privateKey': {
            'type': 'privateKey',
            'value_description': 'private key in ASN.1 format',
            'value': {  # object
                'publicKey': '',  # string
                'privateKey': ''  # string
            }
        },
        'passkey': {
            'type': 'passkey',
            'value_description': 'Password-less login',
            'value': {   # object
                'privateKey': {},     # EC private key in JsonWebKey format (RFC 7517)
                'credentialId': '',   # 32 bytes; base64url encoded
                'signCount': 0,       # we don't currently implement incrementing signCount
                'userId': '',         # supplied by relyingParty at registration; base64url encoded
                'relyingParty': '',   # utf8; i.e. "keepersecurity.com"
                'username': '',       # user.name from webauthn create request
                'createdDate': 0,     # unix timestamp
            }
        }
    }

    # field_values w/o field_type - probably migrated to different types
    # 'pinCode': '',       # string - currently v3.pinCode.type == secret /string/
    # 'keyPair': {         # object - currently v3.keyPair.type == privateKey /string (PEM encoded)/
    #     'publicKey': '', # string
    #     'privateKey': '' # string
    #   },
    # 'note': ''           # string - currently v3.note.type == multiline /string/

    @classmethod
    def is_valid_field(cls, field_json):
        ft_dict = {}
        if field_json:
            try:
                ft_dict = json.loads(field_json)
            except:
                ft_dict = {}

        ft = ft_dict.get('type')
        fv = ft_dict.get('value') or []
        valid_type = RecordV3.is_valid_field_type(ft)
        valid_value = RecordV3.is_valid_field_value(ft, fv)
        result = valid_type and valid_value
        return result

    @classmethod
    def is_valid_field_type(cls, field_type):
        result = True if field_type and cls.field_types.get(field_type) else False
        return result

    @classmethod
    def is_valid_field_value(cls, field_type, field_value):
        result = False
        if field_type and RecordV3.is_valid_field_type(field_type):
            # empty value is OK
            if field_value == []:
                result = True

            if not result and isinstance(field_value, list):
                fdef = cls.field_types.get(field_type) or {}
                ftyp = fdef.get('type')
                fvt = cls.field_values.get(ftyp)
                fval = fvt.get('value')
                results = True
                if ftyp in ('fileRef', 'cardRef', 'addressRef'):
                    for fv in field_value:
                        results = results and RecordV3.is_valid_ref_uid(fv)
                elif isinstance(fval, int):
                    for fv in field_value:
                        results = results and (isinstance(fv, int) or bool(re.match(r'^\s*[-+]?\s*\d+\s*$', str(fv))))
                elif isinstance(fval, str):
                    for fv in field_value:
                        results = results and isinstance(fv, str) and bool(fv)
                elif isinstance(fval, tuple):
                    for fv in field_value:
                        results = results and fval.__contains__(str(fv).strip())
                elif isinstance(fval, dict):
                    for fv in field_value:
                        results = results and isinstance(fv, dict)
                        if not results: break
                        for fvv in fv:
                            val1 = fval.get(fvv)
                            val2 = fv.get(fvv)
                            res = bool(val2)
                            if isinstance(val1, int):
                                res = res and (
                                            isinstance(val2, int) or bool(re.match(r'^\s*[-+]?\s*\d+\s*$', str(val2))))
                            elif isinstance(val1, str):
                                res = res and isinstance(val2, str) and bool(val2)
                            elif isinstance(val1, tuple):
                                res = res and val1.__contains__(str(val2).strip())
                            else:
                                res = False
                            results = results and res
                else:
                    results = False
                result = results

        return result

    @classmethod
    def is_valid_field_data(cls, field_data, required=False):
        errors = []

        # ex. {"type": "name", "value": [{"first": "", "middle": "", "last": ""}]}
        ft = field_data if isinstance(field_data, dict) else RecordV3.record_type_to_dict(field_data)
        ftype = ft.get('type')
        fvalue = ft.get('value') or []
        reqd = required or bool(ft.get('required')) or False
        if ftype and RecordV3.is_valid_field_type(ftype):
            # empty value is OK for non-required fields
            if not fvalue:
                if reqd:
                    errors.append('Field missing required value: ' + str(field_data))
                return errors

            if isinstance(fvalue, list):
                fdef = cls.field_types.get(ftype) or {}
                ftyp = fdef.get('type')
                fvt = cls.field_values.get(ftyp) or {}
                fval = fvt.get('value')
                freq = (fvt.get('required') or []) if reqd else []
                # warnings.append('Couldn\'t find required fields for field type: ' + str(ftyp))

                if ftyp in ('fileRef', 'cardRef', 'addressRef'):
                    for fv in fvalue:
                        # if reqd and not fv: errors.append('Missing required Ref UID: ' + fv)
                        if fv and not RecordV3.is_valid_ref_uid(fv):
                            errors.append('Invalid Ref UID: ' + fv)
                elif isinstance(fval, int):
                    for fv in fvalue:
                        # if reqd and not fv: errors.append('Missing required integer value: ' + fv)
                        if not ((isinstance(fv, int) or bool(re.match(r'^\s*[-+]?\s*\d+\s*$', str(fv))))):
                            errors.append('Invalid integer value: ' + fv)
                elif isinstance(fval, str):
                    for fv in fvalue:
                        # if reqd and not fv: errors.append('Missing required string value: ' + fv)
                        if not isinstance(fv, str):  # and bool(fv)
                            errors.append('Invalid string value: ' + fv)
                elif isinstance(fval, tuple):
                    for fv in fvalue:
                        # if reqd and not fv: errors.append('Missing required enum value: ' + fv)
                        if not fval.__contains__(str(fv).strip()):
                            errors.append('Invalid enum value: ' + fv)
                elif isinstance(fval, dict):
                    for fv in fvalue:
                        if not isinstance(fv, dict):
                            errors.append('Invalid object value: ' + fv)
                        if errors: break
                        for fvv in fv:
                            val1 = fval.get(fvv)
                            val2 = fv.get(fvv)
                            res = bool(val2)
                            if reqd and fvv in freq and not res:
                                errors.append('Missing required object field value: ' + fvv)
                            if not errors:
                                if isinstance(val1, int):
                                    if not (
                                    (isinstance(val2, int) or bool(re.match(r'^\s*[-+]?\s*\d+\s*$', str(val2))))):
                                        errors.append('Invalid integer object value: ' + fvv)
                                elif isinstance(val1, str):
                                    if not isinstance(val2, str):  # and bool(val2)
                                        errors.append('Invalid string object value: ' + fvv)
                                elif isinstance(val1, tuple):
                                    if not val1.__contains__(str(val2).strip()):
                                        errors.append('Invalid enum object value: ' + fvv)
                                else:
                                    errors.append('Invalid object value type: ' + fvv)
                else:
                    errors.append('Invalid value type: ' + fval)
            else:
                errors.append('Expected an array of field values: ' + str(ftype))
        else:
            errors.append('Field has unknown type: ' + str(ftype))

        return errors

    @classmethod
    def is_valid_field_type_ref(cls, field_type_json):
        # field ref inside record type definition - ex. {"$ref":"name", "required":true, "label":"placeName"}
        # 2021-04-26 currently the only used options in field ref are - $ref, label, required
        result = False
        if field_type_json:
            try:
                ft = json.loads(field_type_json)
            except:
                ft = {}
            ref = ft.get('$ref')
            result = RecordV3.is_valid_field_type(ref)

            known_keys = ('$ref', 'label', 'required', 'privacyscreen', 'enforcegeneration', 'complexity')
            unknown_keys = [x for x in ft if x.lower() not in known_keys]
            if unknown_keys:
                logging.warning('Unknown attributes in field reference: ' + str(unknown_keys))

        return result

    @classmethod
    def is_valid_field_type_data(cls, field_type_json):
        # field data inside record type - ex. {"type":"name","value":[{"first":"John","last":"Doe"}],"required":true, "label":"personName"}
        # 2021-04-26 currently the only used options in fields are - type, label, required, value[]
        result = False
        if field_type_json:
            try:
                ft = json.loads(field_type_json)
            except:
                ft = {}
            ref = ft.get('type')
            result = True if ref and cls.field_types.get(ref) else False

            known_keys = ('type', 'label', 'required')
            unknown_keys = [x for x in ft if x.lower() not in known_keys]
            if unknown_keys:
                logging.warning('Unknown attributes in field type data: ' + str(unknown_keys))
        return result

    @staticmethod
    def get_custom_list(custom_list):
        # parse a list of key-value pairs - accepted formats: json, csv, list
        custom = []
        error = ''
        if custom_list:
            if type(custom_list) == str:
                if custom_list[0] == '{' and custom_list[-1] == '}':
                    try:
                        custom_json = json.loads(custom_list)
                        for k, v in custom_json.items():
                            custom.append({
                                'name': k,
                                'value': str(v)
                            })
                    except ValueError as e:
                        error = 'Invalid custom fields JSON input for {0}, Error: {1}'.format(custom_list, e)
                else:
                    pairs = custom_list.split(',')
                    for pair in pairs:
                        idx = pair.find(':')
                        if idx > 0:
                            custom.append({
                                'name': pair[:idx].strip(),
                                'value': pair[idx + 1:].strip()
                            })
                        else:
                            error = 'Invalid custom fields input for {0}. Expected: "Key:Value". Got: "{1}"'.format(
                                custom_list, pair)
            elif type(custom_list) == list:
                for c in custom_list:
                    if type(c) == dict:
                        name = c.get('name')
                        value = c.get('value')
                        if name and value:
                            custom.append({'name': name, 'value': value})
        result = {'custom_list': custom, 'error': error}
        return result

    @staticmethod
    def get_record_password(rt_data):
        # Records v3 allow only one password to be stored either in fields[] or in custom[]
        rt = RecordV3.record_type_to_dict(rt_data)
        flds = (rt.get('fields') or []) + (rt.get('custom') or [])
        passwords = [x.get('value') or [] for x in flds if isinstance(x, dict) and x.get('type') == 'password']
        result = passwords[0][0] if passwords and passwords[0] else None
        return result

    @staticmethod
    def get_record_field_value(rt_data, field_name, printable=True):
        rt = RecordV3.record_type_to_dict(rt_data)
        flds = (rt.get('fields') or []) + (rt.get('custom') or [])
        values = [x.get('value') or [] for x in flds if isinstance(x, dict) and x.get('type') == field_name]
        result = values[0][0] if values and values[0] else ''
        if printable:
            result = str(result)
        return result

    @staticmethod
    def get_record_type_name(rt_data):
        result = None

        rt = rt_data if isinstance(rt_data, dict) else {}
        if rt_data and (isinstance(rt_data, str) or isinstance(rt_data, bytes)):
            try:
                rt = json.loads(rt_data or '{}')
            except:
                logging.error(bcolors.FAIL + 'Unable to parse record type JSON: ' + str(rt_data) + bcolors.ENDC)

        if rt and isinstance(rt, dict):
            rtt = rt.get('type')
            if rtt:
                result = rtt
            else:
                logging.error(bcolors.FAIL + 'Unable to find record type type - JSON: ' + str(rt_data) + bcolors.ENDC)

        return result

    @staticmethod
    def get_record_type_definition(params, rt_data):
        result = None

        rt_type = RecordV3.get_record_type_name(rt_data)
        if rt_type:
            rt_def = RecordV3.resolve_record_type_by_name(params, rt_type)
            if rt_def:
                result = rt_def
            else:
                logging.error(bcolors.FAIL + 'Record type definition not found for type: ' + str(rt_type) +
                              ' - to get list of all available record types use: record-type-info -lr' + bcolors.ENDC)

        return result

    @staticmethod
    def get_fileref_location(params, rt_data):
        # lookup for fileRef presence in following order:
        # 1) non-empty fileRef in fields[] 2) in custom 3) RT definition 4) if not found anywhere return 'custom'
        result = ''

        # first search for non-empty fileRef in record data
        rt = rt_data if isinstance(rt_data, dict) else RecordV3.record_type_to_dict(rt_data)
        flds = rt.get('fields') or []
        fref = [x.get('value') or [] for x in flds if isinstance(x, dict) and x.get('type') == 'fileRef']
        if fref:
          result = 'fields'
        else:
          flds = rt.get('custom') or []
          fref = [x.get('value') or [] for x in flds if isinstance(x, dict) and x.get('type') == 'fileRef']
          if fref:
            result = 'custom'

        # next lookup fileRef in RT definition if needed
        if not result:
          rt_def = RecordV3.get_record_type_definition(params, rt_data)
          rtdef = {}
          if rt_def:
            try: rtdef = json.loads(rt_def)
            except: logging.error(bcolors.FAIL + 'Unable to parse record type definition JSON: ' + str(rt_def) + bcolors.ENDC)
          if rtdef:
            has_fref = next((True for x in (rtdef.get('fields') or []) if '$ref' in x and x.get('$ref') == 'fileRef'), False)
            if has_fref:
              result = 'fields'

        # if not found anywhere - use custom
        if not result:
          result = 'custom'

        return result

    @staticmethod
    def get_record_type_title(rt_data):
        result = None

        rt = rt_data if isinstance(rt_data, dict) else {}
        if rt_data and (isinstance(rt_data, str) or isinstance(rt_data, bytes)):
            try:
                rt = json.loads(rt_data or '{}')
            except:
                logging.error(bcolors.FAIL + 'Unable to parse record type JSON: ' + str(rt_data) + bcolors.ENDC)

        if rt and isinstance(rt, dict):
            rtt = rt.get('title')
            if rtt:
                result = rtt
            else:
                logging.error(bcolors.FAIL + 'Unable to find record type title - JSON: ' + str(rt_data) + bcolors.ENDC)

        return result

    @staticmethod
    def resolve_record_type_by_name(params, record_type_name):
        record_type_info = None
        if record_type_name:
            if params.record_type_cache:
                for v in params.record_type_cache.values():
                    dict = json.loads(v)
                    # TODO: Is 'type' case sensitive
                    if dict and dict.get('$id').lower() == record_type_name.lower():
                        record_type_info = v
                        break

        return record_type_info

    @staticmethod
    def change_record_type(params, rt_data, new_rt_name):
        # Converts rt_data (dict or JSON) from one valid record type to another
        # by moving required fields between fields[] and custom[]

        result = {
            'errors': [],
            'warnings': [],
            'record': {}
        }

        r = rt_data if isinstance(rt_data, dict) else {}
        if isinstance(rt_data, str) or isinstance(rt_data, bytes):
            try:
                r = json.loads(rt_data)
            except:
                result['errors'].append('Unable to parse record type data JSON: ' + str(rt_data))

        newrtd = {}
        rt_def = RecordV3.resolve_record_type_by_name(params, new_rt_name)
        if rt_def:
            try:
                newrtd = json.loads(rt_def)
            except:
                result['errors'].append('Unable to parse record type definition JSON: ' + str(rt_def))
        else:
            result['errors'].append('Record type definition not found for type: ' + str(
                new_rt_name) + ' - to get list of all available record types use: record-type-info -lr')
        if result['errors']: return result

        rt = copy.deepcopy(r)
        newf = newrtd.get('fields') or []
        # newf = [x.get('$ref') for x in newf if isinstance(x, dict)]

        existing_fields = (rt.get('fields') or []) + (rt.get('custom') or [])
        new_fields = []
        for fld in newf:
            index = next((i for i, x in enumerate(existing_fields)
                          if fld.get('$ref', '') == x.get('type', '') and fld.get('label', '') == x.get('label', '')), -1)
            if index >= 0:
                new_fields.append(existing_fields.pop(index))
            else:
                f = {'type': fld['$ref'], 'value': []}
                if 'label' in fld:
                    f['label'] = fld['label']
                new_fields.append(f)
        new_custom = [x for x in existing_fields if isinstance(x.get('value'), list) and len(x.get('value')) > 0]

        rt['fields'] = new_fields
        rt['custom'] = new_custom
        rt['type'] = new_rt_name

        if not result['errors']:
            r = rt

        result['record'] = r
        return result

    @staticmethod
    def is_valid_ref_uid(uid: str) -> bool:
        # UID length is 22 and all characters are valid base64 urlsafe characters
        result = bool(re.search('^[A-Za-z0-9-_]{22}$', uid or ''))
        return result

    @staticmethod
    def add_field_label(rtdef, field_labels_to_add, fname):
        if fname not in field_labels_to_add:
            label_gen = (f.get('label') for f in rtdef.get('fields', []) if f.get('$ref') == fname)
            label = next(label_gen, None)
            if label is not None:
                field_labels_to_add[fname] = label

    @staticmethod
    def convert_options_to_json(params, rt_json, rt_def, kwargs):
        # Converts dot notation options string to JSON string representing a valid record type
        # NB! Currently duplicate field types cannot be added or edited using dot notation syntax
        # Use JSON representation for full add/edit capabilites

        result = {
            'errors': [],
            'warnings': [],
            'record': {}
        }

        is_edit = True if rt_json else False

        rt = {}
        if rt_json:
            try:
                rt = json.loads(rt_json or '{}')
            except:
                result['errors'].append('Unable to parse record type JSON: ' + str(rt_json))

        rtdef = {}
        if rt_def:
            try:
                rtdef = json.loads(rt_def)
            except:
                result['errors'].append('Unable to parse record type definition JSON: ' + str(rt_def))
        if result['errors']: return result

        options = kwargs.get('option') or []
        opts = [(x or '').split("=", 1) for x in options]
        if not options and not kwargs.get('custom_list'):
            return result

        # normalize prefixes: f. -> fields., c. -> custom.
        # so f.name.first is treated as duplicate of fields.name.first
        for x in opts:
            if x and x[0]:
                x[0] = re.sub(r'^\s*fields\.', 'fields.', x[0], 1, flags=re.IGNORECASE)
                x[0] = re.sub(r'^\s*custom\.', 'custom.', x[0], 1, flags=re.IGNORECASE)
                x[0] = re.sub(r'^\s*f\.', 'fields.', x[0], 1, flags=re.IGNORECASE)
                x[0] = re.sub(r'^\s*c\.', 'custom.', x[0], 1, flags=re.IGNORECASE)

        # check for duplicate keys or keys with more than one value
        dupes = [x for x in opts if x and len(x) != 2]  # keys with multiple values
        if dupes:
            result['errors'].append('Found keys with multiple values: ' + str(dupes))
        groups = {}  # duplicate key(s)
        for x, *values in opts: groups[x] = 1 + (groups.get(x) or 0)
        multi = [{x: groups[x]} for x in groups if groups[x] > 1]
        if multi:
            result['errors'].append('Found duplicate keys/values: ' + str(multi))
        if result['errors']: return result

        rt_type = next((x[1] for x in opts if x and len(x) == 2 and x[0].lower() == 'type'), None)
        if not rt_type and is_edit and isinstance(rt, dict): rt_type = rt.get('type')
        rt_title = next((x[1] for x in opts if x and len(x) == 2 and x[0].lower() == 'title'), None)
        rt_notes = next((x[1] for x in opts if x and len(x) == 2 and x[0].lower() == 'notes'), None)
        rt_fields = next((x[1] for x in opts if x and len(x) == 2 and x[0].lower() == 'fields'), None)
        rt_custom = next((x[1] for x in opts if x and len(x) == 2 and x[0].lower() == 'custom'), None)

        if not rt_type:
            result['errors'].append('Record types "type" is required')
        if rt_fields or rt_custom:
            result['errors'].append('Array types fields[] and custom[] cannot be assigned directly')
        if result['errors']: return result

        # Top level RT options - unknown attribute(s) generate error
        # All known attribute(s) /from RT definition/ generate а warning and are silently ignored
        tlo = [x for x in opts if x and not x[0].__contains__('.')]
        ignored = ('$id', 'categories', 'description', 'label')  # these are in RT definitions only
        ilist = [x for x in tlo if x and x[0] in ignored]
        if ilist:
            tlo = [x for x in tlo if x not in ilist]
            result['warnings'].append(
                'Removed record type attributes that should be present in record type definitions only: ' + str(ilist))

        ulist = [x for x in tlo if x and x[0].strip() not in ('type', 'title', 'notes')]
        if ulist:
            result['errors'].append('Unknown top level attributes: ' + str(ulist))

        # field type options - ex. -o field.name.first=Jane -o f.name.last=Doe
        flo = [x for x in opts if x and x[0].__contains__('.')]

        # All fields must be either in fields[] or custom[] arrays
        badg = [x for x in flo if x[0].split('.', 1)[0].strip().lower() not in ('fields', 'custom')]
        if badg:
            result['errors'].append('Unknown field group (not fields[] and not custom[]): ' + str(badg))

        # Allow only valid/known field types - field types are case sensitive?
        badf = [x for x in flo if not RecordV3.field_types.__contains__(x[0].split('.', 2)[1].strip())]
        if badf:
            result['errors'].append('Unknown field types: ' + str(badf))
        if result['errors']: return result

        # only one FT of type password allowed in record v3
        pwds = [x for x in flo if
                x and x[0].startswith(('fields.password', 'f.password', 'custom.password', 'c.password'))]
        if len(pwds) > 1:
            result['errors'].append('Error: Only one password allowed per record! ' + str(pwds))
        elif len(pwds) == 1:
            rtdp = next((True for x in (rtdef.get('fields') or []) if '$ref' in x and x.get('$ref') == 'password'),
                        False)
            pwdc = pwds[0][0] and pwds[0][0].lower().startswith('custom.')
            if rtdp:
                if pwdc: result['errors'].append(
                    'Password must be in fields[] section as defined by record type! ' + str(pwds))
            else:
                if not pwdc: result['errors'].append(
                    'Password must be in custom[] section - record type does not allow password in fields[]! ' + str(
                        pwds))
        if result['errors']: return result

        # All fields with prefix f./fields. must be in record type definition
        # Don't move f./fields. not in RT definition to custom[] - undefined order on duplicates, might break scripts
        rtdf = [x.get('$ref') for x in (rtdef.get('fields') or []) if '$ref' in x]
        flds = [x for x in flo if x and x[0].startswith(('fields.', 'f.'))]
        cust = [x for x in flo if x and x[0].startswith(('custom.', 'c.'))]
        badf = [x for x in flds if not x[0].split('.', 2)[1].strip() in rtdf]
        if badf:
            result['errors'].append(
                'This record type doesn\'t allow "fields." prefix for these (move to custom): ' + str(badf))
        # fileRef must use upload-attachment/delete-attachment commands instead
        refs = [x for x in flds + cust if x[0].split('.', 2)[1].strip().lower() == 'fileref']
        if refs:
            result['errors'].append(
                'File reference manipulations are disabled here. Use upload-attachment/delete-attachment commands instead. ' + str(
                    refs))
        if result['errors']: return result

        # edit command: JSON validation before update
        # add command: fields[] must contain all 'required' fields (and required value parts)
        if not is_edit:
            flon = [x[0] for x in flo if x and x[0]]
            reqd = [x.get('$ref') for x in (rtdef.get('fields') or []) if '$ref' in x and 'required' in x]
            for fld in reqd:
                ft = (RecordV3.field_types.get(fld) or {}).get('type')
                ftr = (RecordV3.field_values.get(ft) or {}).get('required') or []
                if ftr:
                    ftr = ['fields.' + fld + '.' + x for x in ftr]
                    ftrm = [x for x in ftr if x not in flon]
                    if ftrm:
                        result['errors'].append('Missing required fields: ' + str(
                            ftrm) + '   Use `rti -lf field_name --example` to generate valid field sample.')
                elif next((f for f in flon if f.split('.')[:2] == ['fields', fld]), None) is None:
                    result['warnings'].append('Couldn\'t find required fields for the field type: ' + str(ft))
        if result['errors']: return result

        # NB! cmdline labels override RT definition labels which override FT definition labels
        r = {}
        if is_edit and rt_type != rt.get('type'):
            res = RecordV3.change_record_type(params, rt_json, rt_type)
            if not res.get('errors'):
                r = res.get('record') or r
            if res.get('errors'): result['errors'].extend(res['errors'])
            if res.get('warnings'): result['warnings'].extend(res['warnings'])
        else:
            r = copy.deepcopy(rt)  # for edited or deleted items
            if not r: r = {'type': rt_type}  # add command
            if rt_title: r['title'] = rt_title
            if is_edit and rt_title == '': r['title'] = ''  # edit: delete title
            if rt_notes: r['notes'] = rt_notes
            if is_edit and rt_notes == '': r['notes'] = ''  # edit: delete notes
            if not 'fields' in r: r['fields'] = []
            if not 'custom' in r: r['custom'] = []
            unique_field_types = [k for k, v in Counter(rtdf).items() if v == 1]
            field_labels_to_add = {}
            for lst in [flds, cust]:
                for f in lst:
                    if f and len(f) == 2:
                        val = f[1]
                        path = f[0].split('.')
                        forc = path[0].strip() if path else ''  # fields or custom
                        fname = path[1].strip() if path and len(path) > 1 else ''  # field name
                        fvname = path[2].strip() if path and len(path) > 2 else ''  # field attribute (if any)
                        if fname:
                            if not forc in r: r[forc] = []  # create fields[] or custom[]
                            fv = next((x for x in r[forc] if isinstance(x, dict) and x.get('type') == fname), {})
                            if not fv:
                                fv = {'type': fname, 'value': []}
                                r[forc].append(fv)
                            if 'value' not in fv: fv['value'] = []
                            elif not isinstance(fv['value'], list): fv['value'] = []
                            if fvname:
                                # NB! required:true/false comes from RT definition and should not be re/set here
                                if fvname.lower() == 'required':
                                    result['warnings'].append(
                                        'Skipped "required" field attribute which comes from record type definition and should not be set here! ' + str(
                                            f))
                                    continue
                                # check if fvname is FT attribute vs FT value object parts - ex. c.name.label vs. c.name.first
                                if fvname.lower() == 'label':
                                    fv['label'] = val
                                elif is_edit and not val:  # delete
                                    if forc in r:
                                        v = next((x.get('value') or [] for x in r[forc] if
                                                  isinstance(x, dict) and x.get('type') == fname), [])
                                        # v = next((x for x in v if isinstance(x, dict) and fvname in x), {})
                                        v = next((x for x in v if isinstance(x, dict)), {})
                                        v.pop(fvname, None)
                                elif bool(val):  # upsert
                                    if forc == 'fields' and fname in unique_field_types:
                                        RecordV3.add_field_label(rtdef, field_labels_to_add, fname)
                                    # v = next((x for x in fv['value'] if isinstance(x, dict) and fvname in x), None)
                                    v = next((x for x in fv['value'] if isinstance(x, dict)), None)
                                    if v:
                                        v[fvname] = val
                                    else:
                                        fv['value'].append({fvname: val})
                                    ok = RecordV3.is_valid_field_value(fname, [{fvname: val}])
                                    if not ok: result['errors'].append(
                                        'Invalid field value: ' + str({fname: [{fvname: val}]}))
                                else:
                                    result['warnings'].append('Skipped empty field value: ' + str(f))
                            else:  # simple value str/int assign directly - ex. c.login=MyLogin
                                if bool(val):
                                    if forc == 'fields' and fname in unique_field_types:
                                        RecordV3.add_field_label(rtdef, field_labels_to_add, fname)
                                    del fv['value'][:]
                                    fv['value'].append(val)
                                elif 'value' in fv and isinstance(fv['value'], list):  # delete
                                    del fv['value'][:]
                    else:
                        result['errors'].append(
                            'Miltiple field values per single option aren\'t allowed. Use multiple options: -o f.name.first=A -o f.name.last=B ' + str(
                                f))
            if len(field_labels_to_add) > 0:
                for field_type, label in field_labels_to_add.items():
                    for rfield in r.get('fields', []):
                        if rfield['type'] in field_labels_to_add and 'label' not in rfield:
                            rfield['label'] = field_labels_to_add[rfield['type']]


        # add command could pass multiple custom options - ex. --custom='{"name1":"value1", "name2":"value: 2,3,4"}'
        # since dot format can't handle duplicate keys we pass these as kwargs['custom_list']
        custom_list = kwargs.get('custom_list') or []
        custom = [{
            'type': 'text',
            'label': x.get('name') or '',
            'value': [x.get('value')] if x.get('value') else []
        } for x in custom_list if x.get('name') or x.get('value')]
        if custom:
            r['custom'] = (r.get('custom') or [])
            if is_edit:
                # update value of existing custom field or insert new one
                for cr in custom:
                    cold = next((x for x in r['custom'] if
                                 x.get('type') == 'text' and x.get('label') == cr.get('label') and cr.get(
                                     'label') is not None), None)
                    if cold:
                        cold['value'] = cr.get('value')
                    else:
                        r['custom'].append(cr)
            else:
                r['custom'] = r['custom'] + custom

        if r and not result['errors']:
            if not r.get('custom'): r.pop('custom', None)
            if not r.get('fields'): r.pop('fields', None)
            rt = r

        if not result['errors']:
            result['record'] = rt

        return result

    @staticmethod
    def values_to_lowerstring(data_json):
        return RecordV3.values_to_string(data_json).lower()

    @staticmethod
    def values_to_string(data_json):
        result = ''

        rt = RecordV3.record_type_to_dict(data_json)
        result += rt.get('title') or ''
        result += rt.get('notes') or ''

        fields = rt.get('fields') or []
        custom = rt.get('custom') or []
        values = [x.get('value') for x in fields + custom if x.get('value')]
        values = [x if not isinstance(x[0], dict) else ';'.join(str(v) for v in x[0].values()) for x in values if
                  len(x) > 0]
        result += str(values)

        return result

    @staticmethod
    def record_type_to_dict(rt_json):
        rt = {}
        if rt_json:
            try:
                rt = json.loads(rt_json or '{}')
            except:
                rt = {}
                logging.error(bcolors.FAIL + 'Unable to parse record type JSON: ' + str(rt_json) + bcolors.ENDC)
        return rt

    @staticmethod
    def update_password(password, rt_json, rt_def):
        # Delete if pass is empty, Upsert if pass is present:
        # Check if there's password field in fields[], custom[] and replace first instance
        # If no password in RT but RT definition has a password field - add to fields[]
        # else add to custom[]
        result = rt_json

        rt = {}
        if rt_json:
            try:
                rt = json.loads(rt_json or '{}')
            except:
                logging.error(bcolors.FAIL + 'Unable to parse record type JSON: ' + str(rt_json) + bcolors.ENDC)

        rtdef = {}
        if rt_def:
            try:
                rtdef = json.loads(rt_def)
            except:
                logging.error(
                    bcolors.FAIL + 'Unable to parse record type definition JSON: ' + str(rt_def) + bcolors.ENDC)

        if not rt or not rtdef:
            logging.error(bcolors.FAIL + 'Failed to update password field!' + bcolors.ENDC)
            return result

        rtt = rt.get('type')
        rtdt = rtdef.get('$id')
        if not rtt or rtt != rtdt:
            logging.error(
                bcolors.FAIL + 'Record type missing or doesn\'t match definition! ' + str([rtt, rtdt]) + bcolors.ENDC)
            return result

        # Look for existing password in fields[] then custom[] and replace
        fields = rt.get('fields') or []
        custom = rt.get('custom') or []
        rtfp = [x for x in fields if 'type' in x and x.get('type') == 'password']
        rtcp = [x for x in custom if 'type' in x and x.get('type') == 'password']
        changed = False
        if rtfp:
            rtfp[0]['value'] = [password] if password else []
            changed = True
        elif rtcp:
            rtcp[0]['value'] = [password] if password else []
            changed = True
        elif password:
            # no existing password - add to fields[] (if RT definition allows) or to custom[]
            rtdp = next((True for x in (rtdef.get('fields') or []) if '$ref' in x and x.get('$ref') == 'password'),
                        False)
            if rtdp:
                if 'fields' not in rt:
                    rt['fields'] = []
                rt['fields'].append({'type': 'password', 'value': [password]})
            else:
                if 'custom' not in rt:
                    rt['custom'] = []
                rt['custom'].append({'type': 'password', 'value': [password]})
            changed = True

        if changed:
            result = json.dumps(rt)

        return result

    @staticmethod
    def get_field_types():
        ftypes = [{**RecordV3.field_types.get(fkey), **RecordV3.field_values.get(vkey)}
                  for fkey in RecordV3.field_types
                  for vkey in RecordV3.field_values
                  if (RecordV3.field_types.get(fkey) or {}).get('type') == vkey
                  ]
        rows = []  # (id, type, lookup, multiple, description)
        for ft in ftypes:
            field_id = ft.get('$id') or ''
            field_type = ft.get('type') or ''
            lookup = ft.get('lookup') or ''
            multiple = ft.get('multiple') or ''
            description = ft.get('value_description') or ''
            rows.append((field_id, field_type, lookup, multiple, description))
        return rows

    @staticmethod
    def get_field_type(id):
        STR_VALUE = 'text'
        ftypes = [{**RecordV3.field_types.get(fkey), **RecordV3.field_values.get(vkey)}
                  for fkey in RecordV3.field_types
                  for vkey in RecordV3.field_values
                  if (RecordV3.field_types.get(fkey) or {}).get('type') == vkey
                  ]
        ids = [ft for ft in ftypes if id and (id == ft.get('$id'))]
        result = ids[0] if ids else {}
        if result:
            val = result.get('value')
            vtype = ('integer' if isinstance(val, int)
                     else 'string' if isinstance(val, str)
            else 'enum' if isinstance(val, tuple)
            else 'object' if isinstance(val, dict)
            else 'unknown')
            obj = val if vtype != 'object' else {
                x: val[x] if not isinstance(val[x], tuple) else " | ".join(str(t) for t in val[x])
                for x in val
            }
            obj_sample = val if vtype != 'object' else {
                x: 0 if isinstance(val[x], int)
                else STR_VALUE if isinstance(val[x], str)
                else val[x][0] if isinstance(val[x], tuple)
                else val
                for x in val
            }
            value = (0 if vtype == 'integer'
                     else '' if vtype == 'string'
            else val[0] if vtype == 'enum'
            else obj if vtype == 'object'
            else '')
            sample = (0 if vtype == 'integer'
                      else STR_VALUE if vtype == 'string'
            else val[0] if vtype == 'enum'
            else obj_sample if vtype == 'object'
            else '')

            result = {
                'id': result.get('$id') or '',
                'type': result.get('type') or '',
                'valueType': vtype,
                'value': value,
                'sample': sample
            }
        return result

    @staticmethod
    def get_record_type_example(params, rt_name: str) -> str:
        STR_VALUE = 'text'

        result = ''
        rte = {}
        rt_def = RecordV3.resolve_record_type_by_name(params, rt_name)
        if rt_def:
            rtdd = RecordV3.record_type_to_dict(rt_def)
            rtdf = rtdd.get('fields') or []

            rte = {
                'type': rt_name,
                'title': STR_VALUE,
                'notes': STR_VALUE,
                'fields': [],
                'custom': []
            }

            rtdef = {}
            try:
                rtdef = json.loads(rt_def)
            except:
                logging.error(
                    bcolors.FAIL + 'Unable to parse record type definition JSON: ' + str(rt_def) + bcolors.ENDC)

            fields = rtdef.get('fields') or []
            fields = [x.get('$ref') for x in fields]
            for fname in fields:
                ft = RecordV3.get_field_type(fname)
                if not ft:
                    ft = {
                        'id': fname,
                        'type': 'text',
                        'valueType': 'string',
                        'value': '',
                        'sample': 'unknown type'
                    }

                val = {
                    'type': fname,
                    'value': []
                }

                if fname not in ('fileRef', 'addressRef', 'cardRef'):
                    # Fix for webVault - fails if phone's region is not valid country code
                    if fname == 'phone' and 'sample' in ft and 'region' in ft['sample']:
                        ft['sample']['region'] = 'US'

                    val['value'].append(ft.get('sample'))
                    required = next((x.get('required') for x in rtdf if x.get('$ref') == fname), None)
                    if required:
                        val['required'] = required
                    label = next((x.get('label') for x in rtdf if x.get('$ref') == fname), None)
                    if label:
                        val['label'] = label

                rte['fields'].append(val)
        else:
            logging.error(bcolors.FAIL + 'Unable to find record type definition for type: ' + str(rt_name) +
                          '   To show available record types definitions use `record-type-info --list-record *` command' + bcolors.ENDC)

        result = json.dumps(rte, indent=2) if rte else ''
        return result

    @staticmethod
    def display(r, **kwargs):
        record_uid = r['record_uid']
        # print('{0:>20s}: https://keepersecurity.com/vault#detail/{1}'.format('Link', record_uid))
        print('{0:>20s}: {1:<20s}'.format('UID', record_uid))
        # if 'version' in r: print('{0:>20s}: {1:<20s}'.format('Version', str(r['version'])))
        data = {}
        if 'data_unencrypted' in r:
            data = r['data_unencrypted'].decode() if isinstance(r['data_unencrypted'], bytes) else r['data_unencrypted']
            data = json.loads(data)
        fields = data.get('fields') or []
        custom = data.get('custom') or []

        record_type = data['type'] if 'type' in data else ''
        print('{0:>20s}: {1:<20s}'.format('Type', record_type))
        print('{0:>20s}: {1:<20s}'.format('Title', str(data['title']) if 'title' in data else ''))
        # NB! General notes here - fields[] might provide their own notes
        if 'notes' in data:
            notes = data['notes']
            if notes:
                lines = notes.split('\n')
                for i, line in enumerate(lines):
                    print('{0:>21s} {1}'.format('Notes:' if i == 0 else '', line.strip()))
        # fields[] * print Field# NN - atrib: value
        unmask = kwargs.get('unmask', False)
        for c in fields + custom:
            ftyp = c.get('type') or ''
            flab = c.get('label') or ''
            flds = c.get('value') or []
            fval = flds
            fkey = '{} ({})'.format(flab, ftyp)

            if ftyp in ['name', 'securityQuestion', 'paymentCard', 'host', 'keyPair', 'bankAccount', 'phone']:
                if not isinstance(flds, list):
                    flds = [flds]
                fval = []
                for x in flds:
                    if isinstance(x, dict):
                        is_empty = not any(True for k in x.values() if k)
                        if is_empty:
                            continue
                        if ftyp == 'name':
                            fval.append(vault.TypedField.export_name_field(x))
                        elif ftyp == 'securityQuestion':
                            fval.append(vault.TypedField.export_q_and_a_field(x))
                        elif ftyp == 'paymentCard':
                            n = x.get('cardNumber') or ''
                            if n and not unmask:
                                n = '*' + n[-4:]
                            e = x.get('cardExpirationDate') or ''
                            c = (x.get('cardSecurityCode') or '') if unmask else '***'
                            fval.append(f'{n} exp:{e} cvv:{c}')
                        elif ftyp == 'keyPair':
                            public_key = x.get('publicKey')
                            if public_key:
                                fval.append(public_key)
                            private_key = x.get('privateKey')
                            if private_key:
                                fval.append(private_key if unmask else '********')
                        elif ftyp == 'host':
                            fval.append(vault.TypedField.export_host_field(x))
                        elif ftyp == 'phone':
                            fval.append(vault.TypedField.export_phone_field(x))
                        elif ftyp == 'bankAccount':
                            account_type = x.get('accountType') or ''
                            routing_number = x.get('routingNumber') or ''
                            if routing_number and not unmask:
                                routing_number = '*' + routing_number[-3:]
                            account_number = x.get('accountNumber') or ''
                            if account_number and not unmask:
                                account_number = '*' + account_number[-3:]
                            number = f'{account_type}: ' if account_type else ''
                            if routing_number or account_number:
                                if routing_number and account_number:
                                    number += f'{routing_number} / {account_number}'
                                else:
                                    number = routing_number if routing_number else account_number
                            fval.append(number)

                for i, line in enumerate(fval):
                    print('{0:>20s}: {1:<s}'.format(fkey if i == 0 else '', line))
            else:
                if isinstance(flds, list) and len(flds) == 1:
                    if not flds[0]: continue
                    if isinstance(flds[0], dict):
                        if (ftyp not in record_types.RecordFields or ftyp in ('passkey')) and not unmask:
                            fval = '********'
                        else:
                            fval = json.dumps(flds[0], indent=2)
                    elif RecordV3.get_field_type(ftyp).get('type') == 'date' and bool(
                            re.match('^[+-]?[0-9]+$', str(flds[0]).strip())):
                        dt = datetime.datetime.fromtimestamp(int(flds[0] / 1000), tz=datetime.timezone.utc)
                        fval = str(dt.date())
                    elif record_type == 'ssnCard' and ftyp == 'accountNumber' and flab == 'identityNumber':
                        fval = flds[0] if unmask else '********'
                    else:
                        fval = flds[0]
                if fval:
                    if ftyp in ('fileRef', 'cardRef', 'addressRef'):
                        RecordV3.display_ref(ftyp, fval, **kwargs)
                    else:
                        is_masked = (ftyp not in record_types.RecordFields or
                                     ftyp in ['password', 'pinCode', 'secret', 'note', 'oneTimeCode']) and not unmask
                        if is_masked:
                            print('{0:>20s}: {1:<s}'.format(fkey, '********'))
                        else:
                            if isinstance(fval, str):
                                lines = fval.split('\n')
                                lines = [x.rstrip() for x in lines if x]
                                for i, line in enumerate(lines):
                                    print('{0:>20s}: {1}'.format(fkey if i == 0 else '', line))
                            else:
                                print('{0:>20s}: {1:<s}'.format(fkey, str(fval)))

        totp = next((t.get('value') for t in fields if t.get('type', '') == 'oneTimeCode'), None)
        if totp:
            totp = totp[0] if isinstance(totp, list) else totp
        if totp:
            result = get_totp_code(totp)
            if result:
                code, remain, _ = result
                if code: print('{0:>20s}: {1:<20s} valid for {2} sec'.format('Two Factor Code', code, remain))

    @staticmethod
    def display_ref(ftype, fvalue, **kwargs):
        # fileRef can hold multiple references - convert single value to list for compatibility
        fvalue = fvalue if isinstance(fvalue, list) else [fvalue]
        params = kwargs.get('params')
        unmask = kwargs.get('unmask', False)
        if ftype == 'fileRef':
            for fuid in fvalue:
                frec = params.record_cache.get(fuid) or {}
                fdat = frec.get('data_unencrypted') or '{}'
                fdic = RecordV3.record_type_to_dict(fdat)
                title = str(fdic.get('title') or '')
                name = str(fdic.get('name') or '')
                if title:
                    name = title + f' ({name})'
                size = fdic.get('size') or ''
                if isinstance(size, str) and size.isdigit():
                    size = int(size)
                size = HumanBytes.format(size or 0) if isinstance(size, int) else size
                print('{0:>20s}: {1:>22s}   {2:<12s}   {3:<s}'.format(str(ftype), str(fuid), str(size), str(name)))
        elif ftype == 'cardRef':
            for fuid in fvalue:
                frec = params.record_cache.get(fuid) or {}
                fdat = frec.get('data_unencrypted') or '{}'
                fdic = RecordV3.record_type_to_dict(fdat)
                title = RecordV3.get_record_type_title(fdat) or ''
                name = RecordV3.get_record_field_value(fdat, 'text', False) or ''
                card_line = 'Name: ' + name
                card = RecordV3.get_record_field_value(fdat, 'paymentCard', False)
                if isinstance(card, dict):
                    n = card.get('cardNumber') or ''
                    if n and not unmask:
                        n = '*' + n[-4:]
                    e = card.get('cardExpirationDate') or ''
                    c = (card.get('cardSecurityCode') or '') if unmask else '***'
                    card_line += f' | Card: {n} exp:{e} cvv:{c}'
                print('{0:>20s}: {1}'.format('card.' + str(title), card_line))
        elif ftype == 'addressRef':
            for fuid in fvalue:
                frec = params.record_cache.get(fuid) or {}
                fdat = frec.get('data_unencrypted') or '{}'
                fdic = RecordV3.record_type_to_dict(fdat)
                title = RecordV3.get_record_type_title(fdat) or ''
                addr = RecordV3.get_record_field_value(fdat, 'address', False) or '{}'
                addr = addr if isinstance(addr, dict) else RecordV3.record_type_to_dict(addr)
                addr_line = ' | '.join(str(x) for x in addr.values())
                print('{0:>20s}: {1:<s}'.format('address.' + str(title), str(addr_line)))
        else:
            logging.error(
                bcolors.FAIL + 'Unknown field reference type: ' + str(ftype) + ' for ' + str(fvalue) + bcolors.ENDC)

    @staticmethod
    def validate_access(params: KeeperParams, ruid: str):
        if ruid:
            v3_enabled = False
            if params and params.settings and isinstance(params.settings.get('record_types_enabled'), bool):
                v3_enabled = params.settings.get('record_types_enabled')
            if not v3_enabled:
                if params and params.record_cache and ruid in params.record_cache:
                    rv = params.record_cache[ruid].get('version') or None
                    if rv and rv in (3, 4):
                        raise TypeError('Record ' + ruid + ' not found. You don\'t have Record Types enabled.')

    @staticmethod
    def custom_options_to_list(options_list: str) -> list:
        custom = []
        if options_list:
            if type(options_list) == str:
                if options_list[0] == '{' and options_list[-1] == '}':
                    try:
                        custom_json = json.loads(options_list)
                        for k, v in custom_json.items():
                            custom.append({
                                'name': k,
                                'value': str(v)
                            })
                    except ValueError as e:
                        raise TypeError('Invalid custom fields JSON input: {0}'.format(e))
                else:
                    pairs = options_list.split(',')
                    for pair in pairs:
                        idx = pair.find(':')
                        if idx > 0:
                            custom.append({
                                'name': pair[:idx].strip(),
                                'value': pair[idx + 1:].strip()
                            })
                        else:
                            raise TypeError(
                                'Invalid custom fields input. Expected: "Key:Value". Got: "{0}"'.format(pair))

            elif type(options_list) == list:
                for c in options_list:
                    if isinstance(c, dict):
                        name = c.get('name')
                        value = c.get('value')
                        if name and value:
                            custom.append({
                                'name': name,
                                'value': value
                            })
        return custom

    @staticmethod
    def get_title(cached_record):   # type: (Dict[str, Any]) -> str
        if isinstance(cached_record, dict):
            return json.loads(cached_record.get('data_unencrypted', b'{}')).get('title', '')
        else:
            raise TypeError('Record data must be of type dict')



class HumanBytes:
    # Human-readable formatting of bytes, using binary (powers of 1024) or metric (powers of 1000) representation.
    METRIC_LABELS = ["bytes", "kB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"]
    BINARY_LABELS = ["bytes", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB", "ZiB", "YiB"]
    PRECISION_OFFSETS = [0.5, 0.05, 0.005, 0.0005]
    PRECISION_FORMATS = ["{}{:.0f} {}", "{}{:.1f} {}", "{}{:.2f} {}", "{}{:.3f} {}"]

    @classmethod
    def format(cls, num, metric=False, precision=1) -> str:
        assert isinstance(precision, int) and precision >= 0 and precision <= 3, "precision must be an int (range 0-3)"
        unit_labels = cls.METRIC_LABELS if metric else cls.BINARY_LABELS
        last_label = unit_labels[-1]
        unit_step = 1000 if metric else 1024
        unit_step_thresh = unit_step - cls.PRECISION_OFFSETS[precision]

        is_negative = num < 0
        if is_negative:
            num = abs(num)
        if num < unit_step:  # return exact bytes when size is too small
            return cls.PRECISION_FORMATS[0].format('-' if is_negative else '', num, unit_labels[0])
        for unit in unit_labels:
            if num < unit_step_thresh:
                break
            if unit != last_label:
                num /= unit_step
        return cls.PRECISION_FORMATS[precision].format('-' if is_negative else '', num, unit)
