import django_migrate_sql.operations
from django.db import migrations


class Migration(migrations.Migration):

    dependencies = [
        ("arches_controlled_lists", "0008_ensure_languages_in_sync"),
    ]

    operations = [
        migrations.operations.special.SeparateDatabaseAndState(
            state_operations=[
                django_migrate_sql.operations.CreateSQL(
                    name="__arches_migrate_collections_to_clm",
                    sql="\ncreate or replace function __arches_migrate_collections_to_clm(\n    collection_names text[] default null, -- one or more collections to be migrated to controlled lists\n    host text default 'http://localhost:8000/plugins/controlled-list-manager/item/',\n    overwrite boolean default FALSE,\n    preferred_sort_language text default 'en'\n)\nreturns text as $$\ndeclare failed_collections text[];\n    collection text;\n    listitems_to_update_with_multiple_values uuid[];\nbegin\n    -- RDM Collections to Controlled Lists & List Items Migration --\n    -- To use, run: \n    --      select * from __arches_migrate_collections_to_clm(\n    --          ARRAY['Getty AAT', 'http://vocab.getty.edu/aat'],\n    --          'http://localhost:8000/plugins/controlled-list-manager/item/',\n    --          True,\n    --          'en'\n    --       );\n    -- where the input array values are concept prefLabels or identifiers and the optional language is used for sorting\n    -- for collections that contain an apostrophe, use two single quotes, e.g. 'John''s list'\n\n    -- Conceptually:\n    --      a collection becomes a list\n    --      a concept belonging to a collection becomes a list item\n    --      a prefLabel and any altLabels for a concept become list item values\n    --      a concept with no sortorder value will sorted alphabetically around its siblings \n    --          under a given parent by the concept's best prefLabel\n    --      a concept with an existing sortorder value will be sorted ahead of any siblings with no sortorder value\n    --      a concept that participates in multiple collections will have distinct list items for each new list it belongs to\n\n    --      in the RDM concepts are sorted alphabetically, but list items are explicitly ordered using sortorder...\n    --      sort order is calculated at the list level and ordered alphabetically within each leaf of the hierarchy\n\n    -- Check if collection_names are provided\n    if collection_names is null or array_length(collection_names, 1) = 0 then\n        raise exception 'No collection names or identifiers provided.';\n    end if;\n\n    -- Check if input collection names or identifiers exist in the database\n    failed_collections := array(\n        select names\n        from unnest(collection_names) as names\n        where names not in (\n            select value \n            from values v\n            left join concepts c on c.conceptid = v.conceptid\n            where c.nodetype = 'Collection' and\n                (v.valuetype = 'prefLabel' or\n                v.valuetype = 'identifier')\n        )\n    );\n    \n    -- If all provided names do not match any collections, end operation\n    if array_length(collection_names, 1) = array_length(failed_collections, 1) then\n        raise exception 'Failed to find the following collections in the database: %', array_to_string(failed_collections, ', ')\n        using hint = 'Please ensure the provided name or identifier matches a valid collection';\n    end if;\n    \n    -- Remove user provided values from collection_names if they aren't a collection (identifier or prefLabel)\n    if array_length(failed_collections, 1) > 0 then\n        raise warning 'Failed to find the following collections in the database: %', array_to_string(failed_collections, ', ');\n        collection_names := array(\n            select array_agg(elem)\n            from unnest(collection_names) elem\n            where elem <> all(failed_collections)\n        );\n    end if;\n\n    -- If overwrite flag is provided, completely recreate the list/items/values\n    if overwrite then\n        delete from arches_controlled_lists_listitemvalue\n        where list_item_id in (\n            select id\n            from arches_controlled_lists_listitem\n            where list_id in (\n                select id\n                from arches_controlled_lists_list\n                where name = any(collection_names)\n            )\n        );\n\n        delete from arches_controlled_lists_listitem\n        where list_id in (\n            select id\n            from arches_controlled_lists_list\n            where name = any(collection_names)\n        );\n\n        delete from arches_controlled_lists_list\n        where name = any(collection_names);\n    end if;\n\n    -- Migrate Collection -> Controlled List\n    insert into arches_controlled_lists_list (\n        id,\n        name,\n        dynamic,\n        searchable\n    )\n    with identifier_conceptids as (\n        select c.conceptid\n        from concepts c\n        full join values v on\n            c.conceptid = v.conceptid\n        where nodetype = 'Collection' and\n            v.valuetype = 'identifier' and\n            value = ANY(collection_names)\n        )\n    select c.conceptid as id,\n        value as name,\n        false as dynamic,\n        false as searchable\n    from concepts c\n    full join values v on\n        c.conceptid = v.conceptid\n    where nodetype = 'Collection' and\n        v.valuetype = 'prefLabel' and\n        (\n            c.conceptid in (select * from identifier_conceptids) or\n            value = ANY(collection_names)\n        );\n\n    -- Migrate Concepts participating in Collections -> Controlled List Items & Controlled List Item Values\n\n    -- The recursive CTE below is used to assign the conceptid of the list at the root to each concept to be migrated\n    -- On each recursion, it checks if the child (aka conceptidto in relations table) is a parent for another concept.\n    -- The results are stored in a temporary table to avoid re-running non-filtered recursion (done on the whole relations table)\n    -- We keep track of the hierarchy path in order to account for concepts that participate in multiple collections\n    \n    create temporary table temp_collection_hierarchy as\n        with recursive collection_hierarchy as (\n            select conceptidfrom as root_list,\n                conceptidto as child,\n                ARRAY[conceptidfrom] AS path\n            from relations\n            where not exists (\n                select 1 from relations r2 where r2.conceptidto = relations.conceptidfrom\n            ) and relationtype = 'member'\n            union all\n            select ch.root_list,\n                r.conceptidto,\n                ch.path || r.conceptidfrom\n            from collection_hierarchy ch\n            join relations r on ch.child = r.conceptidfrom\n            where relationtype = 'member'\n        )\n        select * from collection_hierarchy;\n    \n    -- This temp table is used to stage list items and values \n    create temporary table temp_list_items_and_values (\n        list_item_id uuid,\n        sortorder bigint,\n        list_id uuid,\n        parent_id uuid,\n        legacy_conceptid uuid,\n        listitemvalue_id uuid,\n        listitemvalue text,\n        listitemvalue_languageid text,\n        listitemvalue_valuetype text,\n        rownumber int\n    );\n\n    -- Build the new hierarchies at the list level, mainly to account for concepts that participate in multiple collections\n    -- then stash results in temp table for preprocessing before inserting into CLM tables\n    foreach collection in array collection_names loop\n        with filtered_collection_hierarchy as (\n            select * \n            from temp_collection_hierarchy\n            where root_list in (select id from arches_controlled_lists_list where name = collection)\n        ),\n        items_with_sortorder as (\n            select ch.child,\n                v.value as sortorder\n            from filtered_collection_hierarchy ch\n            join values v on v.conceptid = ch.child\n            where v.valuetype = 'sortorder'\n        ),\n        -- Rank prefLabels by user provided language, \n        -- if no prefLabel in that language exists for a concept, fall back on next prefLabel ordered by languageid\n        ranked_prefLabels as (\n            select ch.root_list,\n                ch.child,\n                iso.sortorder::int,\n                v.languageid,\n                v.value,\n                ROW_NUMBER() OVER (PARTITION BY ch.child ORDER BY (v.languageid = preferred_sort_language) DESC, languages.id) AS language_rank,\n                r.conceptidfrom,\n                ch.path\n            from filtered_collection_hierarchy ch\n            left join values v on v.conceptid = ch.child\n            left join relations r on r.conceptidto = ch.child\n            left join languages on v.languageid = languages.code\n            left join items_with_sortorder iso on iso.child = ch.child\n            where v.valuetype = 'prefLabel'\n                and r.relationtype = 'member'\n                and r.conceptidfrom in (select unnest(path) from filtered_collection_hierarchy)\n        ),\n        -- Once we've found the best prefLabel for each concept, we can use it to alphabetically sort concepts that don't have a sortorder value\n        -- prioritizing concepts that do have an existing sortorder value\n        alpha_sorted_list_item_hierarchy as (\n            select child as id,\n                row_number() over (partition by conceptidfrom order by sortorder, LOWER(value)) - 1 as sortorder,\n                case when conceptidfrom = root_list then null -- list items at top of hierarchy have no parent list item\n                    else conceptidfrom\n                end as parent_id,\n                root_list as list_id\n            from ranked_prefLabels rpl\n            where language_rank = 1\n        )\n        insert into temp_list_items_and_values (\n            list_item_id,\n            sortorder,\n            list_id,\n            parent_id,\n            legacy_conceptid,\n            listitemvalue_id,\n            listitemvalue,\n            listitemvalue_languageid,\n            listitemvalue_valuetype\n        )\n        select lih.id as list_item_id,\n            lih.sortorder,\n            lih.list_id,\n            lih.parent_id,\n            lih.id as legacy_conceptid,\n            v.valueid as listitemvalue_id,\n            v.value,\n            v.languageid,\n            v.valuetype\n        from alpha_sorted_list_item_hierarchy lih\n        join values v on v.conceptid = lih.id\n        where valuetype in (\n            select valuetype from d_value_types where category in ('note', 'label')\n        );\n    end loop;\n\n    -- Assign row number to help identify concepts that participate in multiple collections\n    -- or exist already as listitems and therefore need new listitem_id's and listitemvalue_id's\n    with assign_row_num as (\n        select list_item_id,\n            sortorder,\n            list_id,\n            parent_id,\n            existing_item,\n            ROW_NUMBER() OVER (PARTITION BY list_item_id ORDER BY existing_item DESC, sortorder ASC) as init_rownumber\n        from (\n            select list_item_id,\n                sortorder,\n                list_id,\n                parent_id,\n                FALSE as existing_item\n            from temp_list_items_and_values\n            union all\n            select id as list_item_id,\n                sortorder,\n                list_id,\n                parent_id,\n                TRUE as existing_item\n            from arches_controlled_lists_listitem\n            ) as t\n    )\n    update temp_list_items_and_values t\n    set rownumber = init_rownumber\n    from assign_row_num a\n    where t.list_item_id = a.list_item_id \n        and t.list_id = a.list_id;\n\n    -- For concepts that participate in multiple collections, mint new listitem_id's and listitemvalue_id's\n    -- However, if a concept needs a new listitem_id, and has multiple values associated with it, ensure that\n    -- the new listitem_id is the same for all listitemvalues\n    listitems_to_update_with_multiple_values := array(\n        select list_item_id\n        from temp_list_items_and_values\n        where rownumber > 1\n        group by list_item_id\n        having count(*) > 1\n    );\n\n    with new_list_item_ids as (\n        select legacy_list_item_id,\n            uuid_generate_v4() as new_list_item_id\n        from unnest(listitems_to_update_with_multiple_values) as t(legacy_list_item_id)\n    )\n    update temp_list_items_and_values t\n    set list_item_id = new_list_item_id\n    from new_list_item_ids n\n    where t.list_item_id = n.legacy_list_item_id\n        and rownumber > 1;\n\n    -- Update list_item_ids for items that don't have multiple values (like prefLabel)\n    if array_length(listitems_to_update_with_multiple_values, 1) > 0 then \n        update temp_list_items_and_values\n        set list_item_id = uuid_generate_v4()\n        where rownumber > 1\n            and legacy_conceptid != any(listitems_to_update_with_multiple_values)\n            and list_item_id = legacy_conceptid;\n    else\n        update temp_list_items_and_values\n        set list_item_id = uuid_generate_v4()\n        where rownumber > 1\n            and list_item_id = legacy_conceptid;\n    end if;\n\n    -- Update listitemvalue_ids\n    update temp_list_items_and_values\n    set listitemvalue_id = uuid_generate_v4()\n    where rownumber > 1;\n    \n    insert into arches_controlled_lists_listitem (\n        id,\n        uri,\n        sortorder,\n        guide,\n        list_id,\n        parent_id\n    )\n    select distinct on (list_item_id, list_id)\n        list_item_id,\n        host || legacy_conceptid as uri,\n        sortorder,\n        false as guide,\n        list_id,\n        parent_id\n    from temp_list_items_and_values;\n\n    -- Migrate concept values -> controlled list item values\n    insert into arches_controlled_lists_listitemvalue (\n        id,\n        value,\n        list_item_id,\n        languageid,\n        valuetype_id\n    )\n    select listitemvalue_id,\n        listitemvalue,\n        list_item_id,\n        listitemvalue_languageid,\n        listitemvalue_valuetype\n    from temp_list_items_and_values;\n    \n    drop table if exists temp_collection_hierarchy;\n    drop table if exists temp_list_items_and_values;\n\n    return format('Collection(s) %s migrated to controlled list(s)', array_to_string(collection_names, ', '));\nend;\n$$ language plpgsql volatile;",
                    reverse_sql="drop function __arches_migrate_collections_to_clm;",
                ),
            ],
        ),
        django_migrate_sql.operations.AlterSQL(
            name="__arches_migrate_collections_to_clm",
            sql="\ncreate or replace function __arches_migrate_collections_to_clm(\n    collection_names text[] default null, -- one or more collections to be migrated to controlled lists\n    host text default 'http://localhost:8000/plugins/controlled-list-manager/item/',\n    overwrite boolean default FALSE,\n    preferred_sort_language text default 'en'\n)\nreturns text as $$\ndeclare failed_collections text[];\n    collection text;\n    listitems_to_update_with_multiple_values uuid[];\nbegin\n    -- RDM Collections to Controlled Lists & List Items Migration --\n    -- To use, run: \n    --      select * from __arches_migrate_collections_to_clm(\n    --          ARRAY['Getty AAT', 'http://vocab.getty.edu/aat'],\n    --          'http://localhost:8000/plugins/controlled-list-manager/item/',\n    --          True,\n    --          'en'\n    --       );\n    -- where the input array values are concept prefLabels or identifiers and the optional language is used for sorting\n    -- for collections that contain an apostrophe, use two single quotes, e.g. 'John''s list'\n\n    -- Conceptually:\n    --      a collection becomes a list\n    --      a concept belonging to a collection becomes a list item\n    --      a prefLabel and any altLabels for a concept become list item values\n    --      a concept with no sortorder value will sorted alphabetically around its siblings \n    --          under a given parent by the concept's best prefLabel\n    --      a concept with an existing sortorder value will be sorted ahead of any siblings with no sortorder value\n    --      a concept that participates in multiple collections and/or exists at n-locations in the collection\n    -- \t\t\twill have distinct list items for each occurrence in the hierarchy beyond the first\n\n    --      in the RDM concepts are sorted alphabetically, but list items are explicitly ordered using sortorder...\n    --      sort order is calculated at the list level and ordered alphabetically within each leaf of the hierarchy\n\n    -- Check if collection_names are provided\n    if collection_names is null or array_length(collection_names, 1) = 0 then\n        raise exception 'No collection names or identifiers provided.';\n    end if;\n\n    -- Check if input collection names or identifiers exist in the database\n    failed_collections := array(\n        select names\n        from unnest(collection_names) as names\n        where names not in (\n            select value \n            from values v\n            left join concepts c on c.conceptid = v.conceptid\n            where c.nodetype = 'Collection' and\n                (v.valuetype = 'prefLabel' or\n                v.valuetype = 'identifier')\n        )\n    );\n    \n    -- If all provided names do not match any collections, end operation\n    if array_length(collection_names, 1) = array_length(failed_collections, 1) then\n        raise exception 'Failed to find the following collections in the database: %', array_to_string(failed_collections, ', ')\n        using hint = 'Please ensure the provided name or identifier matches a valid collection';\n    end if;\n    \n    -- Remove user provided values from collection_names if they aren't a collection (identifier or prefLabel)\n    if array_length(failed_collections, 1) > 0 then\n        raise warning 'Failed to find the following collections in the database: %', array_to_string(failed_collections, ', ');\n        collection_names := array(\n            select array_agg(elem)\n            from unnest(collection_names) elem\n            where elem <> all(failed_collections)\n        );\n    end if;\n\n    -- If overwrite flag is provided, completely recreate the list/items/values\n    if overwrite then\n        delete from arches_controlled_lists_listitemvalue\n        where list_item_id in (\n            select id\n            from arches_controlled_lists_listitem\n            where list_id in (\n                select id\n                from arches_controlled_lists_list\n                where name = any(collection_names)\n            )\n        );\n\n        delete from arches_controlled_lists_listitem\n        where list_id in (\n            select id\n            from arches_controlled_lists_list\n            where name = any(collection_names)\n        );\n\n        delete from arches_controlled_lists_list\n        where name = any(collection_names);\n    end if;\n\n    -- Migrate Collection -> Controlled List\n    insert into arches_controlled_lists_list (\n        id,\n        name,\n        dynamic,\n        searchable\n    )\n    with identifier_conceptids as (\n        select c.conceptid\n        from concepts c\n        full join values v on\n            c.conceptid = v.conceptid\n        where nodetype = 'Collection' and\n            v.valuetype = 'identifier' and\n            value = ANY(collection_names)\n        )\n    select c.conceptid as id,\n        value as name,\n        false as dynamic,\n        false as searchable\n    from concepts c\n    full join values v on\n        c.conceptid = v.conceptid\n    where nodetype = 'Collection' and\n        v.valuetype = 'prefLabel' and\n        (\n            c.conceptid in (select * from identifier_conceptids) or\n            value = ANY(collection_names)\n        );\n\n    -- Migrate Concepts participating in Collections -> Controlled List Items\n\n    create temporary table temp_collection_hierarchy as\n    -- The recursive CTE below is used to assign the conceptid of the list at the root to each concept to be migrated\n    -- On each recursion, it checks if the child (aka conceptidto in relations table) is a parent for another concept.\n    -- The results are stored in a temporary table to avoid re-running non-filtered recursion (done on the whole relations table)\n    -- We keep track of the hierarchy path in order to account for concepts that participate in multiple collections\n    with recursive collection_hierarchy as (\n        select conceptidfrom as root_list,\n            conceptidto as child,\n            ARRAY[conceptidfrom] AS path,\n            conceptidfrom as parent_id\n        from relations r\n        where relationtype = 'member'\n            and conceptidfrom in (select conceptid from concepts where nodetype = 'Collection')\n        union all\n        select ch.root_list,\n            r.conceptidto,\n            ch.path || r.conceptidfrom,\n            r.conceptidfrom as parent_id\n        from collection_hierarchy ch\n        join relations r on ch.child = r.conceptidfrom\n        where relationtype = 'member'\n    ),\n    -- Filter out any collections that are not intended to be migrated in this execution\n    filtered_hierarchy as (\n        select\n            root_list as list_id,\n            child as list_item_id,\n            parent_id,\n            path,\n            v.value::int as sortorder\n        from collection_hierarchy ch\n        left join values v on v.conceptid = ch.child and v.valuetype = 'sortorder'\n        where root_list in (select id from arches_controlled_lists_list where name = ANY(collection_names))\n    ),\n    -- Assign row number to help identify concepts that participate in multiple collections\n    -- or exist already as listitems and therefore need new listitem_id's and listitemvalue_id's\n    assign_row_num as (\n        select list_item_id,\n            sortorder,\n            list_id,\n            parent_id,\n            existing_item_id,\n            path,\n            ROW_NUMBER() OVER (\n                PARTITION BY list_item_id\n                ORDER BY existing_item_id NULLS LAST, sortorder ASC, list_id, parent_id\n            ) as num_concept_occurrence\n        from (\n            select fh.list_item_id,\n                fh.sortorder,\n                fh.list_id,\n                fh.parent_id,\n                fh.path,\n                coalesce(l.id, li.id) as existing_item_id\n            from filtered_hierarchy fh\n            left join arches_controlled_lists_list l on fh.list_item_id = l.id\n            left join arches_controlled_lists_listitem li on fh.list_item_id = li.id \n        ) as t\n    ),\n    -- For items that occur in multiple places in the hierarchy or already exist as list items\n    -- we need to create a new list item for each occurrence of that node in the concept graph\n    mint_new_ids as (\n        select\n            list_item_id as legacy_list_item_id,\n            case \n                when existing_item_id is not null or num_concept_occurrence > 1 then uuid_generate_v4()\n                else list_item_id\n            end as list_item_id,\n            list_id,\n            parent_id,\n            path,\n            sortorder\n        from assign_row_num\n    ),\n    -- Make sure we can point back to the original concept records so we can easily get its associated values\n    -- and join prefLabels to begin process of creating sortorder for list items that don't have it\n    new_items_with_best_label as (\n        select mni.list_item_id,\n            mni.list_id,\n            case\n                when parent_crosswalk.list_item_id = mni.list_id then null -- list items at top of hierarchy have no parent list item\n                else parent_crosswalk.list_item_id -- map to correct parent if new id was minted for it\n            end as parent_id,\n            mni.legacy_list_item_id,\n            mni.parent_id as legacy_parent_id,\n            mni.sortorder,\n            v.languageid,\n            v.value as prefLabel\n        from mint_new_ids mni\n        left join mint_new_ids as parent_crosswalk \n            on mni.parent_id = parent_crosswalk.legacy_list_item_id\n            and mni.list_id = parent_crosswalk.list_id\n            -- ensure we're getting the right lineage by comparing the path of the parent & child(minus last element)\n            and mni.path[1:array_length(mni.path, 1)-1] = parent_crosswalk.path\n        left join (\n            -- Get the best prefLabel for each list item based on user supplied preferred_sort_language / fall back\n            select conceptid, value, languageid, valuetype,\n                ROW_NUMBER() OVER (\n                    PARTITION BY conceptid\n                    ORDER BY (v.languageid = preferred_sort_language) DESC, languages.id\n                ) AS list_item_language_rank\n            from values v\n            left join languages on v.languageid = languages.code\n            where valuetype = 'prefLabel'\n        ) v on mni.legacy_list_item_id = v.conceptid and v.list_item_language_rank = 1\n    )\n    -- Calculate sortorder relative to siblings\n    -- and join URI's stored as identifiers if they exist\n    -- and crosswalk `collector` label to `guide` flag\n    select list_item_id,\n        list_id,\n        parent_id,\n        legacy_list_item_id,\n        legacy_parent_id,\n        ROW_NUMBER() OVER (\n            PARTITION BY parent_id, list_id\n            ORDER BY sortorder NULLS LAST, lower(prefLabel)\n        )-1 AS sortorder,\n        prefLabel,\n        identifier.value as uri,\n        guide.value as guide\n    from new_items_with_best_label\n    left join (\n        select conceptid, value\n        from values\n        where valuetype = 'identifier'\n    ) identifier on legacy_list_item_id = identifier.conceptid\n    left join (\n        select conceptid, value\n        from values\n        where valuetype = 'collector'\n    ) guide on legacy_list_item_id = guide.conceptid;\n\n\n    insert into arches_controlled_lists_listitem (\n        id,\n        uri,\n        sortorder,\n        guide,\n        list_id,\n        parent_id\n    )\n    select\n        list_item_id,\n        case when uri is not null then uri\n            else host || legacy_list_item_id\n        end as uri,\n        sortorder,\n        case when guide is not null then True\n            else False\n        end as guide,\n        list_id,\n        parent_id\n    from temp_collection_hierarchy;\n    \n\n    -- Migrate concept values -> controlled list item values\n    insert into arches_controlled_lists_listitemvalue (\n        id,\n        value,\n        list_item_id,\n        languageid,\n        valuetype_id\n    )\n    select\n        uuid_generate_v4() as id,\n        v.value,\n        tch.list_item_id,\n        v.languageid,\n        v.valuetype\n    from temp_collection_hierarchy tch\n    join values v on tch.legacy_list_item_id = v.conceptid\n    where valuetype in (\n        select valuetype from d_value_types where category in ('note', 'label')\n    );\n    \n    drop table if exists temp_collection_hierarchy;\n\n    return format('Collection(s) %s migrated to controlled list(s)', array_to_string(collection_names, ', '));\nend;\n$$ language plpgsql volatile;",
            reverse_sql="\ncreate or replace function __arches_migrate_collections_to_clm(\n    collection_names text[] default null, -- one or more collections to be migrated to controlled lists\n    host text default 'http://localhost:8000/plugins/controlled-list-manager/item/',\n    overwrite boolean default FALSE,\n    preferred_sort_language text default 'en'\n)\nreturns text as $$\ndeclare failed_collections text[];\n    collection text;\n    listitems_to_update_with_multiple_values uuid[];\nbegin\n    -- RDM Collections to Controlled Lists & List Items Migration --\n    -- To use, run: \n    --      select * from __arches_migrate_collections_to_clm(\n    --          ARRAY['Getty AAT', 'http://vocab.getty.edu/aat'],\n    --          'http://localhost:8000/plugins/controlled-list-manager/item/',\n    --          True,\n    --          'en'\n    --       );\n    -- where the input array values are concept prefLabels or identifiers and the optional language is used for sorting\n    -- for collections that contain an apostrophe, use two single quotes, e.g. 'John''s list'\n\n    -- Conceptually:\n    --      a collection becomes a list\n    --      a concept belonging to a collection becomes a list item\n    --      a prefLabel and any altLabels for a concept become list item values\n    --      a concept with no sortorder value will sorted alphabetically around its siblings \n    --          under a given parent by the concept's best prefLabel\n    --      a concept with an existing sortorder value will be sorted ahead of any siblings with no sortorder value\n    --      a concept that participates in multiple collections will have distinct list items for each new list it belongs to\n\n    --      in the RDM concepts are sorted alphabetically, but list items are explicitly ordered using sortorder...\n    --      sort order is calculated at the list level and ordered alphabetically within each leaf of the hierarchy\n\n    -- Check if collection_names are provided\n    if collection_names is null or array_length(collection_names, 1) = 0 then\n        raise exception 'No collection names or identifiers provided.';\n    end if;\n\n    -- Check if input collection names or identifiers exist in the database\n    failed_collections := array(\n        select names\n        from unnest(collection_names) as names\n        where names not in (\n            select value \n            from values v\n            left join concepts c on c.conceptid = v.conceptid\n            where c.nodetype = 'Collection' and\n                (v.valuetype = 'prefLabel' or\n                v.valuetype = 'identifier')\n        )\n    );\n    \n    -- If all provided names do not match any collections, end operation\n    if array_length(collection_names, 1) = array_length(failed_collections, 1) then\n        raise exception 'Failed to find the following collections in the database: %', array_to_string(failed_collections, ', ')\n        using hint = 'Please ensure the provided name or identifier matches a valid collection';\n    end if;\n    \n    -- Remove user provided values from collection_names if they aren't a collection (identifier or prefLabel)\n    if array_length(failed_collections, 1) > 0 then\n        raise warning 'Failed to find the following collections in the database: %', array_to_string(failed_collections, ', ');\n        collection_names := array(\n            select array_agg(elem)\n            from unnest(collection_names) elem\n            where elem <> all(failed_collections)\n        );\n    end if;\n\n    -- If overwrite flag is provided, completely recreate the list/items/values\n    if overwrite then\n        delete from arches_controlled_lists_listitemvalue\n        where list_item_id in (\n            select id\n            from arches_controlled_lists_listitem\n            where list_id in (\n                select id\n                from arches_controlled_lists_list\n                where name = any(collection_names)\n            )\n        );\n\n        delete from arches_controlled_lists_listitem\n        where list_id in (\n            select id\n            from arches_controlled_lists_list\n            where name = any(collection_names)\n        );\n\n        delete from arches_controlled_lists_list\n        where name = any(collection_names);\n    end if;\n\n    -- Migrate Collection -> Controlled List\n    insert into arches_controlled_lists_list (\n        id,\n        name,\n        dynamic,\n        searchable\n    )\n    with identifier_conceptids as (\n        select c.conceptid\n        from concepts c\n        full join values v on\n            c.conceptid = v.conceptid\n        where nodetype = 'Collection' and\n            v.valuetype = 'identifier' and\n            value = ANY(collection_names)\n        )\n    select c.conceptid as id,\n        value as name,\n        false as dynamic,\n        false as searchable\n    from concepts c\n    full join values v on\n        c.conceptid = v.conceptid\n    where nodetype = 'Collection' and\n        v.valuetype = 'prefLabel' and\n        (\n            c.conceptid in (select * from identifier_conceptids) or\n            value = ANY(collection_names)\n        );\n\n    -- Migrate Concepts participating in Collections -> Controlled List Items & Controlled List Item Values\n\n    -- The recursive CTE below is used to assign the conceptid of the list at the root to each concept to be migrated\n    -- On each recursion, it checks if the child (aka conceptidto in relations table) is a parent for another concept.\n    -- The results are stored in a temporary table to avoid re-running non-filtered recursion (done on the whole relations table)\n    -- We keep track of the hierarchy path in order to account for concepts that participate in multiple collections\n    \n    create temporary table temp_collection_hierarchy as\n        with recursive collection_hierarchy as (\n            select conceptidfrom as root_list,\n                conceptidto as child,\n                ARRAY[conceptidfrom] AS path\n            from relations\n            where not exists (\n                select 1 from relations r2 where r2.conceptidto = relations.conceptidfrom\n            ) and relationtype = 'member'\n            union all\n            select ch.root_list,\n                r.conceptidto,\n                ch.path || r.conceptidfrom\n            from collection_hierarchy ch\n            join relations r on ch.child = r.conceptidfrom\n            where relationtype = 'member'\n        )\n        select * from collection_hierarchy;\n    \n    -- This temp table is used to stage list items and values \n    create temporary table temp_list_items_and_values (\n        list_item_id uuid,\n        sortorder bigint,\n        list_id uuid,\n        parent_id uuid,\n        legacy_conceptid uuid,\n        listitemvalue_id uuid,\n        listitemvalue text,\n        listitemvalue_languageid text,\n        listitemvalue_valuetype text,\n        rownumber int\n    );\n\n    -- Build the new hierarchies at the list level, mainly to account for concepts that participate in multiple collections\n    -- then stash results in temp table for preprocessing before inserting into CLM tables\n    foreach collection in array collection_names loop\n        with filtered_collection_hierarchy as (\n            select * \n            from temp_collection_hierarchy\n            where root_list in (select id from arches_controlled_lists_list where name = collection)\n        ),\n        items_with_sortorder as (\n            select ch.child,\n                v.value as sortorder\n            from filtered_collection_hierarchy ch\n            join values v on v.conceptid = ch.child\n            where v.valuetype = 'sortorder'\n        ),\n        -- Rank prefLabels by user provided language, \n        -- if no prefLabel in that language exists for a concept, fall back on next prefLabel ordered by languageid\n        ranked_prefLabels as (\n            select ch.root_list,\n                ch.child,\n                iso.sortorder::int,\n                v.languageid,\n                v.value,\n                ROW_NUMBER() OVER (PARTITION BY ch.child ORDER BY (v.languageid = preferred_sort_language) DESC, languages.id) AS language_rank,\n                r.conceptidfrom,\n                ch.path\n            from filtered_collection_hierarchy ch\n            left join values v on v.conceptid = ch.child\n            left join relations r on r.conceptidto = ch.child\n            left join languages on v.languageid = languages.code\n            left join items_with_sortorder iso on iso.child = ch.child\n            where v.valuetype = 'prefLabel'\n                and r.relationtype = 'member'\n                and r.conceptidfrom in (select unnest(path) from filtered_collection_hierarchy)\n        ),\n        -- Once we've found the best prefLabel for each concept, we can use it to alphabetically sort concepts that don't have a sortorder value\n        -- prioritizing concepts that do have an existing sortorder value\n        alpha_sorted_list_item_hierarchy as (\n            select child as id,\n                row_number() over (partition by conceptidfrom order by sortorder, LOWER(value)) - 1 as sortorder,\n                case when conceptidfrom = root_list then null -- list items at top of hierarchy have no parent list item\n                    else conceptidfrom\n                end as parent_id,\n                root_list as list_id\n            from ranked_prefLabels rpl\n            where language_rank = 1\n        )\n        insert into temp_list_items_and_values (\n            list_item_id,\n            sortorder,\n            list_id,\n            parent_id,\n            legacy_conceptid,\n            listitemvalue_id,\n            listitemvalue,\n            listitemvalue_languageid,\n            listitemvalue_valuetype\n        )\n        select lih.id as list_item_id,\n            lih.sortorder,\n            lih.list_id,\n            lih.parent_id,\n            lih.id as legacy_conceptid,\n            v.valueid as listitemvalue_id,\n            v.value,\n            v.languageid,\n            v.valuetype\n        from alpha_sorted_list_item_hierarchy lih\n        join values v on v.conceptid = lih.id\n        where valuetype in (\n            select valuetype from d_value_types where category in ('note', 'label')\n        );\n    end loop;\n\n    -- Assign row number to help identify concepts that participate in multiple collections\n    -- or exist already as listitems and therefore need new listitem_id's and listitemvalue_id's\n    with assign_row_num as (\n        select list_item_id,\n            sortorder,\n            list_id,\n            parent_id,\n            existing_item,\n            ROW_NUMBER() OVER (PARTITION BY list_item_id ORDER BY existing_item DESC, sortorder ASC) as init_rownumber\n        from (\n            select list_item_id,\n                sortorder,\n                list_id,\n                parent_id,\n                FALSE as existing_item\n            from temp_list_items_and_values\n            union all\n            select id as list_item_id,\n                sortorder,\n                list_id,\n                parent_id,\n                TRUE as existing_item\n            from arches_controlled_lists_listitem\n            ) as t\n    )\n    update temp_list_items_and_values t\n    set rownumber = init_rownumber\n    from assign_row_num a\n    where t.list_item_id = a.list_item_id \n        and t.list_id = a.list_id;\n\n    -- For concepts that participate in multiple collections, mint new listitem_id's and listitemvalue_id's\n    -- However, if a concept needs a new listitem_id, and has multiple values associated with it, ensure that\n    -- the new listitem_id is the same for all listitemvalues\n    listitems_to_update_with_multiple_values := array(\n        select list_item_id\n        from temp_list_items_and_values\n        where rownumber > 1\n        group by list_item_id\n        having count(*) > 1\n    );\n\n    with new_list_item_ids as (\n        select legacy_list_item_id,\n            uuid_generate_v4() as new_list_item_id\n        from unnest(listitems_to_update_with_multiple_values) as t(legacy_list_item_id)\n    )\n    update temp_list_items_and_values t\n    set list_item_id = new_list_item_id\n    from new_list_item_ids n\n    where t.list_item_id = n.legacy_list_item_id\n        and rownumber > 1;\n\n    -- Update list_item_ids for items that don't have multiple values (like prefLabel)\n    if array_length(listitems_to_update_with_multiple_values, 1) > 0 then \n        update temp_list_items_and_values\n        set list_item_id = uuid_generate_v4()\n        where rownumber > 1\n            and legacy_conceptid != any(listitems_to_update_with_multiple_values)\n            and list_item_id = legacy_conceptid;\n    else\n        update temp_list_items_and_values\n        set list_item_id = uuid_generate_v4()\n        where rownumber > 1\n            and list_item_id = legacy_conceptid;\n    end if;\n\n    -- Update listitemvalue_ids\n    update temp_list_items_and_values\n    set listitemvalue_id = uuid_generate_v4()\n    where rownumber > 1;\n    \n    insert into arches_controlled_lists_listitem (\n        id,\n        uri,\n        sortorder,\n        guide,\n        list_id,\n        parent_id\n    )\n    select distinct on (list_item_id, list_id)\n        list_item_id,\n        host || legacy_conceptid as uri,\n        sortorder,\n        false as guide,\n        list_id,\n        parent_id\n    from temp_list_items_and_values;\n\n    -- Migrate concept values -> controlled list item values\n    insert into arches_controlled_lists_listitemvalue (\n        id,\n        value,\n        list_item_id,\n        languageid,\n        valuetype_id\n    )\n    select listitemvalue_id,\n        listitemvalue,\n        list_item_id,\n        listitemvalue_languageid,\n        listitemvalue_valuetype\n    from temp_list_items_and_values;\n    \n    drop table if exists temp_collection_hierarchy;\n    drop table if exists temp_list_items_and_values;\n\n    return format('Collection(s) %s migrated to controlled list(s)', array_to_string(collection_names, ', '));\nend;\n$$ language plpgsql volatile;",
            state_reverse_sql="drop function __arches_migrate_collections_to_clm;",
        ),
    ]
