{# Copyright 2020-2023 Curtin University
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Author: Richard Hosking #}
{#
The purpose of this script it to export the umatched records to help with identifying up stream data fixed
Primarily, the goal is to create a flat structure which is suitable for graphing in Kibana
#}

WITH google_analytics as (
    {% if google_analytics3_unmatched_table_id %}
    SELECT
        {{ google_analytics3_isbn }} as ISBN,
        title,
        release_date,
        "google_analytics" as source
    FROM `{{ google_analytics3_unmatched_table_id }}`
    {% else %}
    SELECT
        CAST(NULL as STRING) as ISBN,
        CAST(NULL as STRING) as title,
        CAST(NULL as DATE) as release_date,
        "google_analytics" as source
    {% endif %}
),

google_books as (
    {% if google_books_unmatched_table_id %}
    SELECT
        {{ google_books_isbn }} as ISBN,
        title,
        release_date,
        "google_books" as source
    FROM `{{ google_books_unmatched_table_id }}`
    {% else %}
    SELECT
        CAST(NULL as STRING) as ISBN,
        CAST(NULL as STRING) as title,
        CAST(NULL as DATE) as release_date,
        "google_books" as source
    {% endif %}
),

jstor as (
    {% if jstor_unmatched_table_id %}
    SELECT
        {{ jstor_isbn }} as ISBN,
        title,
        release_date,
        "jstor" as source
    FROM `{{ jstor_unmatched_table_id }}`
    {% else %}
    SELECT
        CAST(NULL as STRING) as ISBN,
        CAST(NULL as STRING) as title,
        CAST(NULL as DATE) as release_date,
        "jstor" as source
    {% endif %}
),

irus_oapen as (
    {% if irus_oapen_unmatched_table_id %}
    SELECT
        {{ irus_oapen_isbn }} as ISBN,
        title,
        release_date,
        "irus_oapen" as source
    FROM `{{ irus_oapen_unmatched_table_id }}`
    {% else %}
    SELECT
        CAST(NULL as STRING) as ISBN,
        CAST(NULL as STRING) as title,
        CAST(NULL as DATE) as release_date,
        "irus_oapen" as source
    {% endif %}
),

ucl_discovery as (
    {% if ucl_discovery_unmatched_table_id %}
    SELECT
        {{ ucl_discovery_isbn }} as ISBN,
        title,
        release_date,
        "ucl_discovery" as source
    FROM `{{ ucl_discovery_unmatched_table_id }}`
    {% else %}
    SELECT
        CAST(NULL as STRING) as ISBN,
        CAST(NULL as STRING) as title,
        CAST(NULL as DATE) as release_date,
        "ucl_discovery" as source
    {% endif %}
),

irus_fulcrum as (
    {% if irus_fulcrum_unmatched_table_id %}
    SELECT
        {{ irus_fulcrum_isbn }} as ISBN,
        title,
        release_date,
        "irus_fulcrum" as source
    FROM `{{ irus_fulcrum_unmatched_table_id }}`
    {% else %}
    SELECT
        CAST(NULL as STRING) as ISBN,
        CAST(NULL as STRING) as title,
        CAST(NULL as DATE) as release_date,
        "irus_fulcrum" as source
    {% endif %}
),

all_un_matched as (
SELECT * FROM google_analytics
    UNION ALL
SELECT * FROM google_books
    UNION ALL
SELECT * FROM jstor
    UNION ALL
SELECT * FROM irus_oapen
    UNION ALL
SELECT * FROM irus_fulcrum
    UNION ALL
SELECT * FROM ucl_discovery
)

SELECT
    ISBN,
    release_date,
    ARRAY_AGG(DISTINCT IF(source = "google_analytics", title, NULL) IGNORE NULLS LIMIT 1)[SAFE_OFFSET(0)] as google_analytics_title,
    ARRAY_AGG(DISTINCT IF(source = "google_books", title, NULL) IGNORE NULLS LIMIT 1)[SAFE_OFFSET(0)] as google_books_title,
    ARRAY_AGG(DISTINCT IF(source = "jstor", title, NULL) IGNORE NULLS LIMIT 1)[SAFE_OFFSET(0)] as jstor_title,
    ARRAY_AGG(DISTINCT IF(source = "irus_oapen", title, NULL) IGNORE NULLS LIMIT 1)[SAFE_OFFSET(0)] as irus_oapen_title,
    ARRAY_AGG(DISTINCT IF(source = "ucl_discovery", title, NULL) IGNORE NULLS LIMIT 1)[SAFE_OFFSET(0)] as ucl_discovery_title,
    ARRAY_AGG(DISTINCT IF(source = "irus_fulcrum", title, NULL) IGNORE NULLS LIMIT 1)[SAFE_OFFSET(0)] as irus_fulcrum_title,
    "google_analytics" in UNNEST(ARRAY_AGG(source)) as in_google_analytics,
    "google_books" in UNNEST(ARRAY_AGG(source)) as in_google_books,
    "jstor" in UNNEST(ARRAY_AGG(source)) as in_jstor,
    "irus_oapen" in UNNEST(ARRAY_AGG(source)) as in_irus_oapen,
    "ucl_discovery" in UNNEST(ARRAY_AGG(source)) as in_ucl_discovery,
    "irus_fulcrum" in UNNEST(ARRAY_AGG(source)) as in_irus_fulcrum,
FROM all_un_matched
WHERE ISBN IS NOT NULL
GROUP BY ISBN, release_date