{# Copyright 2020-2023 Curtin University
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Author: Richard Hosking #}

# Helper Function:
{#
Output Schema:
name      STRING    NULLABLE
value     INTEGER   NULLABLE
#}
CREATE TEMP FUNCTION group_items_google_analytics3(items ARRAY<STRUCT<name STRING, value INT64>>) as (
  ARRAY(
    (SELECT AS STRUCT
      name,
      SUM(value) as value,
    FROM UNNEST(items)
    GROUP BY name)
  )
);

# Helper Function:
{#
Output Schema:
Country_of_Sale      STRING    NULLABLE
qty                  INTEGER   NULLABLE
#}
CREATE TEMP FUNCTION group_items_google_books_sales(items ARRAY<STRUCT<Country_of_Sale STRING, qty INT64>>) as (
  ARRAY(
    (SELECT AS STRUCT
      Country_of_Sale,
      SUM(qty) as qty,
    FROM UNNEST(items)
    GROUP BY Country_of_Sale)
  )
);

# Helper Function:
{#
Output Schema:
Country_name            STRING    NULLABLE
Total_Item_Requests     INTEGER   NULLABLE
#}
CREATE TEMP FUNCTION group_items_jstor_country(items ARRAY<STRUCT<Country_name STRING, Total_Item_Requests INT64>>) as (
  ARRAY(
    (SELECT AS STRUCT
      Country_name,
      SUM(Total_Item_Requests) as Total_Item_Requests,
    FROM UNNEST(items)
    GROUP BY Country_name)
  )
);

# Helper Function:
{#
Output Schema:
Institution             STRING    NULLABLE
Total_Item_Requests     INTEGER   NULLABLE
#}
CREATE TEMP FUNCTION group_items_jstor_institution(items ARRAY<STRUCT<Institution STRING, Total_Item_Requests INT64>>) as (
  ARRAY(
    (SELECT AS STRUCT
      Institution,
      SUM(Total_Item_Requests) as Total_Item_Requests,
    FROM UNNEST(items)
    GROUP BY Institution)
  )
);

# Helper Function:
{#
Output Schema:
name                            STRING    NULLABLE
code                            STRING    NULLABLE
title_requests                  INTEGER   NULLABLE
total_item_investigations       INTEGER   NULLABLE
total_item_requests             INTEGER   NULLABLE
unique_item_investigations      INTEGER   NULLABLE
unique_item_requests            INTEGER   NULLABLE
#}
CREATE TEMP FUNCTION group_items_irus_country(items ARRAY<STRUCT<name STRING, code STRING, title_requests INT64, total_item_investigations INT64, total_item_requests INT64, unique_item_investigations INT64, unique_item_requests INT64>>) as (
  ARRAY(
    (SELECT AS STRUCT
      name,
      MAX(code) as code,
      SUM(title_requests) as title_requests,
      SUM(total_item_investigations) as total_item_investigations,
      SUM(total_item_requests) as total_item_requests,
      SUM(unique_item_investigations) as unique_item_investigations,
      SUM(unique_item_requests) as unique_item_requests
    FROM UNNEST(items)
    GROUP BY name)
  )
);

# Helper Function:
{#
Output Schema:
name                            STRING    NULLABLE
code                            STRING    NULLABLE
total_item_investigations       INTEGER   NULLABLE
total_item_requests             INTEGER   NULLABLE
unique_item_investigations      INTEGER   NULLABLE
unique_item_requests            INTEGER   NULLABLE
#}
CREATE TEMP FUNCTION group_items_irus_fulcrum_country(items ARRAY<STRUCT<name STRING, code STRING, total_item_investigations INT64, total_item_requests INT64, unique_item_investigations INT64, unique_item_requests INT64>>) as (
  ARRAY(
    (SELECT AS STRUCT
      name,
      MAX(code) as code,
      SUM(total_item_investigations) as total_item_investigations,
      SUM(total_item_requests) as total_item_requests,
      SUM(unique_item_investigations) as unique_item_investigations,
      SUM(unique_item_requests) as unique_item_requests
    FROM UNNEST(items)
    GROUP BY name)
  )
);

# Helper Function:
{#
Output Schema:
latitude                        FLOAT    NULLABLE
longitude                       FLOAT    NULLABLE
city                            STRING    NULLABLE
country_name                    STRING    NULLABLE
country_code                    STRING    NULLABLE
title_requests                  INTEGER   NULLABLE
total_item_investigations       INTEGER   NULLABLE
total_item_requests             INTEGER   NULLABLE
unique_item_investigations      INTEGER   NULLABLE
unique_item_requests            INTEGER   NULLABLE
#}
CREATE TEMP FUNCTION group_items_irus_location(items ARRAY<STRUCT<latitude FLOAT64, longitude FLOAT64, city STRING, country_name STRING, country_code STRING, title_requests INT64, total_item_investigations INT64, total_item_requests INT64, unique_item_investigations INT64, unique_item_requests INT64>>) as (
  ARRAY(
    (SELECT AS STRUCT
      MAX(latitude) as latitude,
      MAX(longitude) as longitude,
      city,
      MAX(country_name) as country_name,
      MAX(country_code) as country_code,
      SUM(title_requests) as title_requests,
      SUM(total_item_investigations) as total_item_investigations,
      SUM(total_item_requests) as total_item_requests,
      SUM(unique_item_investigations) as unique_item_investigations,
      SUM(unique_item_requests) as unique_item_requests
    FROM UNNEST(items)
    GROUP BY city)
  )
);

# Helper Function:
{#
Output Schema:
name                            STRING    NULLABLE
code                            STRING    NULLABLE
namew                           INTEGER   NULLABLE
downloads                       INTEGER   NULLABLE
#}
CREATE TEMP FUNCTION group_items_worldreader_country(items ARRAY<STRUCT<country_name STRING, code STRING, downloads INT64>>) as (
  ARRAY(
    (SELECT AS STRUCT
      country_name,
      MAX(code) as country_code,
      SUM(downloads) as downloads
    FROM UNNEST(items)
    GROUP BY country_name)
  )
);

# Helper Function:
{#
Output Schema:
lower_trim_word                 STRING   NULLABLE
#}
CREATE TEMP FUNCTION custom_split(str STRING, sep STRING) AS (
  ARRAY(
    SELECT LOWER(TRIM(word)) 
    FROM UNNEST(SPLIT(IFNULL(str,''),sep)) AS word
    WHERE TRIM(word) != ''
    )
);

{#
The purpose of this block of SQL is to create an empty row of data, which comforms to the OAPEN raw data.

Defined in the create_oaebu_book_product_table method, in onix_workflow.py, is the value of 'irus_oapen_table_id'.
This will either point to 'empty_irus_oapen' (the name of this SQL block) or the name of the real data table in bigquery.
The reason for the choice of selecting an empty row, is that some partners will not have corresponding data to query.
Providng an empty row enable simplicity of the downstream queries and also means the resulting schema across all publishers is the same.
#}
WITH empty_irus_oapen as (
    SELECT
        CAST(null as STRING) as ISBN,
        CAST(null as DATE) as release_date,
        CAST(null as STRING) as book_title,
        CAST(null as STRING) as publisher,
        CAST(null as STRING) as version,
        CAST(null as INT64) as title_requests,
        CAST(null as INT64) as total_item_investigations,
        CAST(null as INT64) as total_item_requests,
        CAST(null as INT64) as unique_item_investigations,
        CAST(null as INT64) as unique_item_requests,
        [STRUCT(
            CAST(null as STRING) as name, CAST(null as STRING) as code, CAST(null as INT64) as title_requests, CAST(null as INT64) as total_item_investigations,
            CAST(null as INT64) as total_item_requests, CAST(null as INT64) as unique_item_investigations, CAST(null as INT64) as unique_item_requests
        )] as country,
        [STRUCT(
            CAST(null as FLOAT64) as latitude, CAST(null as FLOAT64) as longitude, CAST(null as STRING) as city, CAST(null as STRING) as country_name,
            CAST(null as STRING) as country_code, CAST(null as INT64) as title_requests, CAST(null as INT64) as total_item_investigations,
            null as total_item_requests, CAST(null as INT64) as unique_item_investigations, CAST(null as INT64) as unique_item_requests
        )] as locations
),

{#
The purpose of this block of SQL is to create an empty row of data, which comforms to the IRUS Fulcrum raw data.

Defined in the create_oaebu_book_product_table method, in onix_workflow.py, is the value of 'irus_fulcrum_table_id'.
This will either point to 'empty_irus_fulcrum' (the name of this SQL block) or the name of the real data table in bigquery.
The reason for the choice of selecting an empty row, is that some partners will not have corresponding data to query.
Providng an empty row enable simplicity of the downstream queries and also means the resulting schema across all publishers is the same.
#}
empty_irus_fulcrum as (
    SELECT
        CAST(null as STRING) as ISBN,
        CAST(null as DATE) as release_date,
        CAST(null as STRING) as book_title,
        CAST(null as STRING) as publisher,
        CAST(null as INT64) as total_item_investigations,
        CAST(null as INT64) as total_item_requests,
        CAST(null as INT64) as unique_item_investigations,
        CAST(null as INT64) as unique_item_requests,
        [STRUCT(
            CAST(null as STRING) as name, CAST(null as STRING) as code, CAST(null as INT64) as total_item_investigations,
            CAST(null as INT64) as total_item_requests, CAST(null as INT64) as unique_item_investigations, CAST(null as INT64) as unique_item_requests
        )] as country,
),

{#
The purpose of this block of SQL is to create an empty row of data, which comforms to the jstor institution raw data.

Defined in the create_oaebu_book_product_table method, in onix_workflow.py, is the value of 'jstor_institution_table_id'.
This will either point to 'empty_jstor_institution' (the name of this SQL block) or the name of the real data table in bigquery.
The reason for the choice of selecting an empty row, is that some partners will not have corresponding data to query.
Providng an empty row enable simplicity of the downstream queries and also means the resulting schema across all publishers is the same.
#}
empty_jstor_institution as (
    SELECT
        Cast(null as STRING) as eISBN,
        Cast(null as STRING) as ISBN,
        CAST(null as Date) as release_date,
        Cast(null as STRING) as Book_Title,
        Cast(null as STRING) as Book_ID,
        Cast(null as STRING) as Authors,
        Cast(null as INT64) as Copyright_Year,
        Cast(null as STRING) as Disciplines,
        Cast(null as STRING) as Usage_Type,
        Cast(null as STRING) as Institution,
        Cast(null as INT64) as Total_Item_Requests
),

{#
The purpose of this block of SQL is to create an empty row of data, which comforms to the jstor country raw data.

Defined in the create_oaebu_book_product_table method, in onix_workflow.py, is the value of 'jstor_country_table_id'.
This will either point to 'empty_jstor_country' (the name of this SQL block) or the name of the real data table in bigquery.
The reason for the choice of selecting an empty row, is that some partners will not have corresponding data to query.
Providng an empty row enable simplicity of the downstream queries and also means the resulting schema across all publishers is the same.
#}
empty_jstor_country as (
    SELECT
        Cast(null as STRING) as eISBN,
        Cast(null as STRING) as ISBN,
        CAST(null as Date) as release_date,
        Cast(null as STRING) as Book_Title,
        Cast(null as STRING) as Book_ID,
        Cast(null as STRING) as Authors,
        Cast(null as INT64) as Copyright_Year,
        Cast(null as STRING) as Disciplines,
        Cast(null as STRING) as Usage_Type,
        Cast(null as STRING) as Country_Name,
        Cast(null as INT64) as Total_Item_Requests
),

{#
The purpose of this block of SQL is to create an empty row of data, which comforms to the google books traffic raw data.

Defined in the create_oaebu_book_product_table method, in onix_workflow.py, is the value of 'google_books_traffic_table_id'.
This will either point to 'empty_google_books_traffic' (the name of this SQL block) or the name of the real data table in bigquery.
The reason for the choice of selecting an empty row, is that some partners will not have corresponding data to query.
Providng an empty row enable simplicity of the downstream queries and also means the resulting schema across all publishers is the same.
#}
empty_google_books_traffic as (
    SELECT
        CAST(NULL as STRING) as Primary_ISBN,
        CAST(NULL as DATE) as release_date,
        CAST(NULL as STRING) as title,
        CAST(NULL as INT64) as Book_Visits_BV_,
        CAST(NULL as INT64) as Non_Unique_Buy_Clicks,
        CAST(NULL as INT64) as BV_with_Buy_Clicks,
        CAST(NULL as INT64) as Pages_Viewed,
        CAST(NULL as INT64) as BV_with_Pages_Viewed,
        CAST(NULL as FLOAT64) as Buy_Link_CTR
),

{#
The purpose of this block of SQL is to create an empty row of data, which comforms to the google books sales raw data.

Defined in the create_oaebu_book_product_table method, in onix_workflow.py, is the value of 'google_books_sales_table_id'.
This will either point to 'empty_google_books_sales' (the name of this SQL block) or the name of the real data table in bigquery.
The reason for the choice of selecting an empty row, is that some partners will not have corresponding data to query.
Providng an empty row enable simplicity of the downstream queries and also means the resulting schema across all publishers is the same.
#}
empty_google_books_sales as (
    SELECT
        CAST(NULL as STRING) as Primary_ISBN,
        CAST(NULL as STRING) as Imprint_Name,
        CAST(NULL as STRING) as Title,
        CAST(NULL as STRING) as Author,
        CAST(NULL as DATE) as release_date,
        CAST(NULL as INT64) as qty,
        CAST(NULL as STRING) as Country_of_Sale
),

{#
The purpose of this block of SQL is to create an empty row of data, which comforms to the google analytics raw data.

Defined in the create_oaebu_book_product_table method, in onix_workflow.py, is the value of 'google_analytics3_table_id'.
This will either point to 'empty_google_analytics3' (the name of this SQL block) or the name of the real data table in bigquery.
The reason for the choice of selecting an empty row, is that some partners will not have corresponding data to query.
Providng an empty row enable simplicity of the downstream queries and also means the resulting schema across all publishers is the same.
#}
empty_google_analytics3 as (
    SELECT
        CAST(null as STRING) as publication_id,
        CAST(null as DATE) as release_date,
        CAST(null as FLOAT64) as average_time,
        CAST(null as STRING) as publication_whole_or_part,
        CAST(null as STRING) as publication_type,
        CAST(null as STRING) as publication_format,
        STRUCT(
            [STRUCT(CAST(NULL as STRING) as name, CAST(null as INT64) as value)] as country
        ) as {{ ga3_views_field }}
),

empty_internet_archive as (
    SELECT
        CAST(null as STRING) as ISBN13,
        CAST(null as STRING) as title,
        CAST(null as INTEGER) as value,
        CAST(null as DATE) as month,
        CAST(null as DATE) as release_date,
),

empty_worldreader as (
    SELECT
        CAST(null as STRING) as ISBN13,
        CAST(null as STRING) as title,
        CAST(null as STRING) as country_code,
        CAST(null as STRING) as country_name,
        CAST(null as INTEGER) as value,
        CAST(null as DATE) as month,
        CAST(null as DATE) as release_date,
),

{#
The purpose of this block of SQL is to create an empty row of data, which comforms to the UCL Discovery raw data.

Defined in the create_oaebu_book_product_table method, in onix_workflow.py, is the value of 'ucl_discovery_table_id'.
This will either point to 'empty_ucl_discovery' (the name of this SQL block) or the name of the real data table in bigquery.
The reason for the choice of selecting an empty row, is that some partners will not have corresponding data to query.
Providng an empty row enable simplicity of the downstream queries and also means the resulting schema across all publishers is the same.
#}
empty_ucl_discovery as (
    SELECT
        CAST(NULL as STRING) as ISBN,
        CAST(NULL as STRING) as eprint_id,
        CAST(NULL as DATE) as release_date,
        CAST(NULL as INT64) as total_downloads,
        [STRUCT(CAST(NULL as INT64) as count, CAST(NULL as STRING) as value)] as country
),

{#

#}
empty_work_ids as (
    SELECT
        CAST(NULL as STRING) as isbn13,
        CAST(NULL as STRING) as work_id,
),

{#

#}
empty_work_family_ids as (
    SELECT
        CAST(NULL as STRING) as isbn13,
        CAST(NULL as STRING) as work_family_id,
),

{#
This block of SQL queries data in an ONIX release, and produces a normalised and focused dataset to pass into the next step of this workflow.

The 'project_id', 'onix_dataset_id', and 'onix_release_date' are all passed in as JINJA parameters to ensure the correct raw table is selected for each publisher.
The query also only selects for product forms that match: "Digital download and online", "Digital (delivered electronically)", or "Digital download"

Also of note, the onix format stores all the different subject classifications together. This query neatly separates out BIC, BISAC, Thema and keywords to their own lists for easier access.
#}
onix_ebook_titles_raw as (
    SELECT
        ISBN13,
        STRUCT(
            Doi,
            ProductForm,
            EditionNumber,
            # This IF statement is used to select between two fields to extract the title as different publishers place the title value in different locations
            IF(
                onix.TitleDetails[SAFE_OFFSET(0)].TitleElements[SAFE_OFFSET(0)].TitleText is not null,
                onix.TitleDetails[SAFE_OFFSET(0)].TitleElements[SAFE_OFFSET(0)].TitleText,
                onix.TitleDetails[SAFE_OFFSET(0)].TitleElements[SAFE_OFFSET(0)].TitleWithoutPrefix) as title,
            ARRAY(SELECT
                SUBSTRING(CAST(dates.Date as STRING), 0, 4)
            FROM UNNEST(onix.PublishingDates) as dates
            WHERE dates.PublishingDateRole = "Publication date")[SAFE_OFFSET(0)] as published_year,
            # This subquery creates the published_date when the date.Date field is 8 characters long (YYYYMMDD)
            ARRAY(SELECT 
                SAFE.PARSE_DATE("%Y%m%d", SUBSTRING(SAFE_CAST(dates.Date AS STRING), 0, 8))
            FROM UNNEST(onix.PublishingDates) AS dates
            WHERE dates.PublishingDateRole = "Publication date" AND 
                CHAR_LENGTH(SAFE_CAST(dates.Date AS STRING)) = 8 )[SAFE_OFFSET(0)] AS published_date,
            # The following four sub-queries all pull data from the onix.subjects array, but neatly organised the values into the 4 current classification systems
            ARRAY(
                SELECT
                    subject.SubjectCode
                FROM UNNEST(onix.Subjects) as subject
                WHERE subject.SubjectSchemeIdentifier = "BIC_subject_category") as bic_subjects,
            ARRAY(
                SELECT
                    subject.SubjectCode
                FROM UNNEST(onix.Subjects) as subject
                WHERE subject.SubjectSchemeIdentifier = "BISAC_Subject_Heading") as bisac_subjects,
            ARRAY(
                SELECT
                    subject.SubjectCode
                FROM UNNEST(onix.Subjects) as subject
                WHERE subject.SubjectSchemeIdentifier = "Thema_subject_category") as thema_subjects,
            (SELECT 
                custom_split(heading,';') 
            FROM UNNEST(onix.Subjects) as subject, UNNEST(subject.SubjectHeadingText) as heading
            WHERE subject.SubjectSchemeIdentifier = "Keywords") as keywords,
            (SELECT
                ARRAY(
                    SELECT as STRUCT
                        PersonName, PersonNameInverted, ORCID
                    FROM UNNEST(onix.Contributors) as contributor
                )
            ) as authors
        ) as onix
    FROM `{{ onix_table_id }}` as onix
),

{#
The purpose of this query is to dedepulicate any repeated rows as a safely check.
#}
onix_ebook_titles as (
    SELECT
        dedupe.*
    FROM (
        SELECT
            ARRAY_AGG(raw LIMIT 1)[OFFSET(0)] dedupe
        FROM onix_ebook_titles_raw as raw
        GROUP BY ISBN13
    )
),

# Google Analytics
{#
The purpose of this block of SQL is to organise the metrics from google analytics for easier consumption of downstream queries.

Defined in the create_oaebu_book_product_table method, in onix_workflow.py, is the value of 'google_analytics3_table_id'.
This will either point to 'empty_google_analytics3' (defined above as an empty row) or the name of the real data table in bigquery.
The reason for the choice of selecting an empty row, is that some partners will not have corresponding data to query.
Providng an empty row enable simplicity of the downstream queries and also means the resulting schema across all publishers is the same.
#}
google_analytics3_grouped_metrics AS(
  SELECT
    publication_id,
    release_date,
    publication_whole_or_part,
    publication_format,
  IF
    (publication_format = 'PDF'
      AND publication_whole_or_part = 'whole',
      group_items_google_analytics3(ARRAY_CONCAT_AGG({{ ga3_views_field }}.country)),
      ARRAY_AGG(STRUCT(CAST(NULL as STRING) as name, CAST(null as INT64) as value))) AS pdf_book_country,
  IF
    (publication_format = 'PDF'
      AND publication_whole_or_part = 'part',
      group_items_google_analytics3(ARRAY_CONCAT_AGG({{ ga3_views_field }}.country)),
      ARRAY_AGG(STRUCT(CAST(NULL as STRING) as name, CAST(null as INT64) as value))) AS pdf_chapter_country,
  IF
    (publication_format = 'HTML'
      AND publication_whole_or_part = 'whole',
      group_items_google_analytics3(ARRAY_CONCAT_AGG({{ ga3_views_field }}.country)),
      ARRAY_AGG(STRUCT(CAST(NULL as STRING) as name, CAST(null as INT64) as value))) AS html_book_country,
  IF
    (publication_format = 'HTML'
      AND publication_whole_or_part = 'part',
      group_items_google_analytics3(ARRAY_CONCAT_AGG({{ ga3_views_field }}.country)),
      ARRAY_AGG(STRUCT(CAST(NULL as STRING) as name, CAST(null as INT64) as value))) AS html_chapter_country,
  IF
    (publication_format = 'EPUB'
      AND publication_whole_or_part = 'whole',
      group_items_google_analytics3(ARRAY_CONCAT_AGG({{ ga3_views_field }}.country)),
      ARRAY_AGG(STRUCT(CAST(NULL as STRING) as name, CAST(null as INT64) as value))) AS epub_book_country,
  IF
    (publication_format = 'EPUB'
      AND publication_whole_or_part = 'part',
      group_items_google_analytics3(ARRAY_CONCAT_AGG({{ ga3_views_field }}.country)),
      ARRAY_AGG(STRUCT(CAST(NULL as STRING) as name, CAST(null as INT64) as value))) AS epub_chapter_country,
  IF
    (publication_format = 'MOBI'
      AND publication_whole_or_part = 'whole',
      group_items_google_analytics3(ARRAY_CONCAT_AGG({{ ga3_views_field }}.country)),
      ARRAY_AGG(STRUCT(CAST(NULL as STRING) as name, CAST(null as INT64) as value))) AS mobi_book_country,
  IF
    (publication_format = 'MOBI'
      AND publication_whole_or_part = 'part',
      group_items_google_analytics3(ARRAY_CONCAT_AGG({{ ga3_views_field }}.country)),
      ARRAY_AGG(STRUCT(CAST(NULL as STRING) as name, CAST(null as INT64) as value))) AS mobi_chapter_country,
  IF
    (publication_format IN ('PDF','HTML', 'EPUB', 'MOBI')
      AND publication_whole_or_part IN ('whole', 'part'),
      group_items_google_analytics3(ARRAY_CONCAT_AGG({{ ga3_views_field }}.country)),
      ARRAY_AGG(STRUCT(CAST(NULL as STRING) as name, CAST(null as INT64) as value))) AS downloads_total_country,
  IF
    (publication_whole_or_part = '(citation)',
      group_items_google_analytics3(ARRAY_CONCAT_AGG({{ ga3_views_field }}.country)),
      ARRAY_AGG(STRUCT(CAST(NULL as STRING) as name, CAST(null as INT64) as value))) AS views_total_country,
  FROM
    `{{ google_analytics3_table_id }}`
  WHERE
    publication_type = "book"
  GROUP BY
    publication_id,
    release_date,
    publication_whole_or_part,
    publication_format ),

google_analytics3_metrics AS (
  SELECT
    publication_id AS ISBN13,
    release_date,
    STRUCT( ARRAY_CONCAT_AGG(views_total_country) AS views_total_country,
      group_items_google_analytics3(ARRAY_CONCAT_AGG(downloads_total_country)) AS downloads_total_country,
      ARRAY_CONCAT_AGG(pdf_book_country) AS downloads_pdf_book_country,
      ARRAY_CONCAT_AGG(pdf_chapter_country) AS downloads_pdf_chapter_country,
      ARRAY_CONCAT_AGG(html_book_country) AS downloads_html_book_country,
      ARRAY_CONCAT_AGG(html_chapter_country) AS downloads_html_chapter_country,
      ARRAY_CONCAT_AGG(epub_book_country) AS downloads_epub_book_country,
      ARRAY_CONCAT_AGG(epub_chapter_country) AS downloads_epub_chapter_country,
      ARRAY_CONCAT_AGG(mobi_book_country) AS downloads_mobi_book_country,
      ARRAY_CONCAT_AGG(mobi_chapter_country) AS downloads_mobi_chapter_country ) AS metrics
  FROM
    google_analytics3_grouped_metrics
  GROUP BY
    publication_id,
    release_date ),

# Google Books Sales Metrics
{#
The purpose of this block of SQL is to organise the metrics from google book sales for easier consumption of downstream queries.

Defined in the create_oaebu_book_product_table method, in onix_workflow.py, is the value of 'google_books_sales_table_id'.
This will either point to 'empty_google_books_sales' (defined above as an empty row) or the name of the real data table in bigquery.
The reason for the choice of selecting an empty row, is that some partners will not have corresponding data to query.
Providng an empty row enable simplicity of the downstream queries and also means the resulting schema across all publishers is the same.
#}
google_books_sales_metrics as (
    SELECT
        Primary_ISBN as ISBN13,
        release_date,
        STRUCT(SUM(qty) as qty, group_items_google_books_sales(ARRAY_AGG(STRUCT(Country_of_Sale, qty))) as countries) as metrics
    FROM `{{ google_books_sales_table_id }}`
    GROUP BY Primary_ISBN, release_date
),

# Google Books Sales Metadata
{#
The purpose of this block of SQL is to organise the Metadata from google book sales for easier consumption of downstream queries.

Defined in the create_oaebu_book_product_table method, in onix_workflow.py, is the value of 'google_books_sales_table_id'.
This will either point to 'empty_google_books_sales' (defined above as an empty row) or the name of the real data table in bigquery.
The reason for the choice of selecting an empty row, is that some partners will not have corresponding data to query.
Providng an empty row enable simplicity of the downstream queries and also means the resulting schema across all publishers is the same.
#}
google_books_sales_metadata as (
    SELECT
        Primary_ISBN as ISBN13, MAX(Imprint_Name) as Imprint_Name, MAX(Title) as Title, MAX(Author) as Author
    FROM `{{ google_books_sales_table_id }}`
    GROUP BY Primary_ISBN
),

# Google Books Traffic Metrics
{#
The purpose of this block of SQL is to organise the metrics from google book traffic for easier consumption of downstream queries.

Defined in the create_oaebu_book_product_table method, in onix_workflow.py, is the value of 'google_books_traffic_table_id'.
This will either point to 'empty_google_books_traffic' (defined above as an empty row) or the name of the real data table in bigquery.
The reason for the choice of selecting an empty row, is that some partners will not have corresponding data to query.
Providng an empty row enable simplicity of the downstream queries and also means the resulting schema across all publishers is the same.
#}
google_books_traffic_metrics as (
    SELECT
        Primary_ISBN as ISBN13,
        release_date,
        STRUCT(
            SUM(Book_Visits_BV_) as Book_Visits_BV_, SUM(BV_with_Pages_Viewed) as BV_with_Pages_Viewed, SUM(Non_Unique_Buy_Clicks) as Non_Unique_Buy_Clicks,
            SUM(BV_with_Buy_Clicks) as BV_with_Buy_Clicks, SUM(Buy_Link_CTR) as Buy_Link_CTR, SUM(Pages_Viewed) as Pages_Viewed
        ) as metrics
    FROM `{{ google_books_traffic_table_id }}`
    GROUP BY Primary_ISBN, release_date
),

# Google Books Traffic Metadata
{#
The purpose of this block of SQL is to organise the Metadata from google book traffic for easier consumption of downstream queries.

Defined in the create_oaebu_book_product_table method, in onix_workflow.py, is the value of 'google_books_traffic_table_id'.
This will either point to 'empty_google_books_traffic' (defined above as an empty row) or the name of the real data table in bigquery.
The reason for the choice of selecting an empty row, is that some partners will not have corresponding data to query.
Providng an empty row enable simplicity of the downstream queries and also means the resulting schema across all publishers is the same.
#}
google_books_traffic_metadata as (
    SELECT
        Primary_ISBN  as ISBN13, MAX(title) as Title
    FROM `{{ google_books_traffic_table_id }}`
    GROUP BY Primary_ISBN
),

# JSTOR Country Metircs
{#
The purpose of this block of SQL is to organise the metrics from JSTOR country for easier consumption of downstream queries.

Defined in the create_oaebu_book_product_table method, in onix_workflow.py, is the value of 'jstor_country_table_id'.
This will either point to 'empty_jstor_country' (defined above as an empty row) or the name of the real data table in bigquery.
The reason for the choice of selecting an empty row, is that some partners will not have corresponding data to query.
Providng an empty row enable simplicity of the downstream queries and also means the resulting schema across all publishers is the same.
#}
jstor_country_metrics as (
    SELECT
        eISBN as ISBN13, release_date, group_items_jstor_country(ARRAY_AGG(STRUCT(Country_name, Total_Item_Requests))) as metrics
    FROM `{{ jstor_country_table_id }}`
    GROUP BY eISBN, release_date
),

# JSTOR Country Metadata
{#
The purpose of this block of SQL is to organise the Metadata from JSTOR country for easier consumption of downstream queries.

Defined in the create_oaebu_book_product_table method, in onix_workflow.py, is the value of 'jstor_country_table_id'.
This will either point to 'empty_jstor_country' (defined above as an empty row) or the name of the real data table in bigquery.
The reason for the choice of selecting an empty row, is that some partners will not have corresponding data to query.
Providng an empty row enable simplicity of the downstream queries and also means the resulting schema across all publishers is the same.
#}
jstor_country_metadata as (
    SELECT
        eISBN as ISBN13, MAX(Book_Title) as Book_Title,
        MAX(Book_ID) as Book_ID, MAX(Authors) as Authors, MAX(ISBN) as ISBN, eISBN,
        MAX(Copyright_Year) as Copyright_Year, MAX(Disciplines) as Disciplines, MAX(Usage_Type) as Usage_Type
    FROM `{{ jstor_country_table_id }}`
    GROUP BY eISBN
),

# JSTOR Institutions Metircs
{#
The purpose of this block of SQL is to organise the metrics from JSTOR institution for easier consumption of downstream queries.

Defined in the create_oaebu_book_product_table method, in onix_workflow.py, is the value of 'jstor_institution_table_id'.
This will either point to 'empty_jstor_institution' (defined above as an empty row) or the name of the real data table in bigquery.
The reason for the choice of selecting an empty row, is that some partners will not have corresponding data to query.
Providng an empty row enable simplicity of the downstream queries and also means the resulting schema across all publishers is the same.
#}
jstor_institution_metrics as (
    SELECT
        eISBN as ISBN13, release_date, group_items_jstor_institution(ARRAY_AGG(STRUCT(Institution, Total_Item_Requests))) as metrics
    FROM `{{ jstor_institution_table_id }}`
    GROUP BY eISBN, release_date
),

# JSTOR Institutions Metadata
{#
The purpose of this block of SQL is to organise the Metadata from JSTOR institution for easier consumption of downstream queries.

Defined in the create_oaebu_book_product_table method, in onix_workflow.py, is the value of 'jstor_institution_table_id'.
This will either point to 'empty_jstor_institution' (defined above as an empty row) or the name of the real data table in bigquery.
The reason for the choice of selecting an empty row, is that some partners will not have corresponding data to query.
Providng an empty row enable simplicity of the downstream queries and also means the resulting schema across all publishers is the same.
#}
jstor_institution_metadata as (
    SELECT
        eISBN as ISBN13,
        MAX(Book_Title) as Book_Title, MAX(Book_ID) as Book_ID, MAX(Authors) as Authors, MAX(ISBN) as ISBN, eISBN,
        MAX(Copyright_Year) as Copyright_Year, MAX(Disciplines) as Disciplines, MAX(Usage_Type) as Usage_Type
    FROM `{{ jstor_institution_table_id }}`
    GROUP BY eISBN
),

# IRUS OPAEN Metrics
{#
The purpose of this block of SQL is to organise the metrics from IRUS OAPEN for easier consumption of downstream queries.

Defined in the create_oaebu_book_product_table method, in onix_workflow.py, is the value of 'oapen_table_id'.
This will either point to 'empty_irus_oapen' (defined above as an empty row) or the name of the real data table in bigquery.
The reason for the choice of selecting an empty row, is that some partners will not have corresponding data to query.
Providng an empty row enable simplicity of the downstream queries and also means the resulting schema across all publishers is the same.
#}
irus_oapen_metrics as (
    SELECT
        ISBN as ISBN13,
        release_date,
        STRUCT(
            MAX(version) as version, SUM(title_requests) as title_requests, SUM(total_item_investigations) as total_item_investigations,
            SUM(total_item_requests) as total_item_requests, SUM(unique_item_investigations) as unique_item_investigations,
            SUM(unique_item_requests) as unique_item_requests, group_items_irus_country(ARRAY_CONCAT_AGG(country)) as country,
            group_items_irus_location(ARRAY_CONCAT_AGG(locations)) as locations
        ) as metrics
    FROM `{{ irus_oapen_table_id }}`
    GROUP BY ISBN, release_date
),

# IRUS OAPEN Metadata
{#
The purpose of this block of SQL is to organise the Metadata from IRUS OAPEN for easier consumption of downstream queries.

Defined in the create_oaebu_book_product_table method, in onix_workflow.py, is the value of 'irus_oapen_table_id'.
This will either point to 'empty_irus_oapen' (defined above as an empty row) or the name of the real data table in bigquery.
The reason for the choice of selecting an empty row, is that some partners will not have corresponding data to query.
Providng an empty row enable simplicity of the downstream queries and also means the resulting schema across all publishers is the same.
#}
irus_oapen_metadata as (
    SELECT
        ISBN as ISBN13, MAX(book_title) as book_title, MAX(publisher) as publisher
    FROM `{{ irus_oapen_table_id }}`
    GROUP BY ISBN
),

# IRUS Fulcrum Metrics
{#
The purpose of this block of SQL is to organise the metrics from IRUS Fulcrum for easier consumption of downstream queries.

Defined in the create_oaebu_book_product_table method, in onix_workflow.py, is the value of 'irus_fulcrum_table_id'.
This will either point to 'empty_irus_fulcrum' (defined above as an empty row) or the name of the real data table in bigquery.
The reason for the choice of selecting an empty row, is that some partners will not have corresponding data to query.
Providng an empty row enable simplicity of the downstream queries and also means the resulting schema across all publishers is the same.
#}
irus_fulcrum_metrics as (
    SELECT
        ISBN as ISBN13,
        release_date,
        STRUCT(
            SUM(total_item_investigations) as total_item_investigations,
            SUM(total_item_requests) as total_item_requests, SUM(unique_item_investigations) as unique_item_investigations,
            SUM(unique_item_requests) as unique_item_requests, group_items_irus_fulcrum_country(ARRAY_CONCAT_AGG(country)) as country
        ) as metrics
    FROM `{{ irus_fulcrum_table_id }}`
    GROUP BY ISBN, release_date
),

# IRUS Fulcrum Metadata
{#
The purpose of this block of SQL is to organise the Metadata from IRUS OAPEN for easier consumption of downstream queries.

Defined in the create_oaebu_book_product_table method, in onix_workflow.py, is the value of 'irus_fulcrum_table_id'.
This will either point to 'empty_irus_oapen' (defined above as an empty row) or the name of the real data table in bigquery.
The reason for the choice of selecting an empty row, is that some partners will not have corresponding data to query.
Providng an empty row enable simplicity of the downstream queries and also means the resulting schema across all publishers is the same.
#}
irus_fulcrum_metadata as (
    SELECT
        ISBN as ISBN13, MAX(book_title) as book_title, MAX(publisher) as publisher
    FROM `{{ irus_fulcrum_table_id }}`
    GROUP BY ISBN
),

# UCL Discovery Metrics
{#
The purpose of this block of SQL is to organise the metrics from UCL Discovery for easier consumption of downstream queries.

Defined in the create_oaebu_book_product_table method, in onix_workflow.py, is the value of 'ucl_discovery_table_id'.
This will either point to 'empty_ucl_discovery' (defined above as an empty row) or the name of the real data table in bigquery.
The reason for the choice of selecting an empty row, is that some partners will not have corresponding data to query.
Providng an empty row enables simplicity of the downstream queries and also means the resulting schema across all publishers is the same.
#}
ucl_discovery_metrics AS (
    SELECT
        ISBN as ISBN13,
        release_date,
        STRUCT( 
            MAX(total_downloads) AS total_downloads,
            ARRAY_AGG(STRUCT(
                ucl_country.value AS country_code,
                countries.iso_name AS country_name,
                ucl_country.count AS country_downloads)) AS country  
            ) AS metrics
        FROM
            `{{ ucl_discovery_table_id }}`,
            UNNEST(country) AS ucl_country
        LEFT JOIN
            `{{ country_table_id }}` AS countries
        ON
            ucl_country.value = countries.alpha2
        GROUP BY
            ISBN13, release_date
),

# UCL Discovery Metadata
{#
The purpose of this block of SQL is to organise the Metadata from UCL Discovery for easier consumption of downstream queries.

Defined in the create_oaebu_book_product_table method, in onix_workflow.py, is the value of 'irus_fulcrum_table_id'.
This will either point to 'empty_irus_oapen' (defined above as an empty row) or the name of the real data table in bigquery.
The reason for the choice of selecting an empty row, is that some partners will not have corresponding data to query.
Providng an empty row enable simplicity of the downstream queries and also means the resulting schema across all publishers is the same.
#}
ucl_discovery_metadata as (
    SELECT
        ISBN as ISBN13, 
        eprint_id
    FROM `{{ ucl_discovery_table_id }}`
    GROUP BY ISBN13, eprint_id
),

# Worldreader Metrics
worldreader_metrics as (
    SELECT
        ISBN13, 
        release_date,
        STRUCT(
            SUM(value) as total_downloads,
            ARRAY_AGG(STRUCT(
                country_code,
                country_name, 
                value AS downloads
            )) AS country
        ) AS metrics
      #group_items_worldreader_country(ARRAY_CONCAT(country_code, country_name, value)) as country
      #ARRAY_AGG(value) as country_downloads, 
      #ARRAY_AGG(country_name) as country
  FROM `{{ worldreader_table_id }}`
  GROUP BY ISBN13, release_date
),

# Worldreader Metadata
worldreader_metadata as (
    SELECT
        ISBN13, 
        MAX(title) as book_title, 
    FROM `{{ worldreader_table_id }}`
    GROUP BY ISBN13
),

# Internet Archive Metrics
internet_archive_metrics as (
    SELECT 
        ISBN13,
        release_date,
        STRUCT(
            SUM(value) as total_downloads
        ) as metrics
    FROM `{{ internet_archive_table_id }}`
    GROUP BY ISBN13, release_date
),

# Internet Archive Metadata
internet_archive_metadata as (
    SELECT
        ISBN13,
        MAX(title) as book_title
    FROM `{{ internet_archive_table_id }}`
    GROUP BY ISBN13
),


# Crossref Events
{#
The purpose of this block of SQL is to
#}
crossref_events as (
    SELECT
        public_data.isbn as ISBN13,
        LAST_DAY(DATE(CAST(SPLIT(month_source.month, "-")[OFFSET(0)] as INT64), CAST(SPLIT(month_source.month, "-")[OFFSET(1)] as INT64), 1), MONTH) as release_date,
        ARRAY_AGG(STRUCT(month_source.source, month_source.count)) as metrics
    FROM `{{ book_table_id }}` as public_data, UNNEST(public_data.events.months) as month_source
    GROUP BY public_data.isbn, month_source.month
),

{#
The purpose of the block of SQL is to select a unique set of release dates (also read as unique months).
The secondary purpose of this block is to not include months for which no metrics source has any data for a particular release.
This ensures that for any book, there is only an array of months for where at least one source of metrics is available
#}
unique_releases as (
    SELECT
        DISTINCT(release_date) as release_date,
    FROM
        UNNEST(ARRAY_CONCAT(
            ARRAY(SELECT DISTINCT(release_date) FROM crossref_events),
            ARRAY(SELECT DISTINCT(release_date) FROM google_analytics3_metrics),
            ARRAY(SELECT DISTINCT(release_date) FROM crossref_events),
            ARRAY(SELECT DISTINCT(release_date) FROM google_books_sales_metrics),
            ARRAY(SELECT DISTINCT(release_date) FROM google_books_traffic_metrics),
            ARRAY(SELECT DISTINCT(release_date) FROM jstor_country_metrics),
            ARRAY(SELECT DISTINCT(release_date) FROM jstor_institution_metrics),
            ARRAY(SELECT DISTINCT(release_date) FROM irus_oapen_metrics),
            ARRAY(SELECT DISTINCT(release_date) FROM irus_fulcrum_metrics),
            ARRAY(SELECT DISTINCT(release_date) FROM worldreader_metrics),
            ARRAY(SELECT DISTINCT(release_date) FROM internet_archive_metrics)
        )
    ) as release_date
    ORDER BY release_date DESC
),

{#
The purpose of this query is to link the list of books, obtained via the ONIX feed, to a list of unique months.
The result, is a row for each ISBN - release_date (month) comhination.
This is made use of in the following query
#}
ebook_months as (
    SELECT
        ISBN13, release_date
    FROM onix_ebook_titles
    LEFT JOIN unique_releases on 1 = 1
    ORDER BY ISBN13, release_date DESC
),

{#
The purpose of this query is to link all the metrics, which are already indexed by release_date (month) to the set of ISBN-Release_date combinations from the above query

The logic is essentially a series of LEFT JOINs, always matching the metric source to the ebook_months.ISBN13 and ebook_months.release_date fields
Finally, the last line 'GROUP BY ebook_months.ISBN13' groups all the various months into a single row per ISBN, with an ARRAY of STRUCTs
These STRUCTS (one per month) contain the 'month' as a field, and subfeilds contains all the metrics from each source.
#}
metrics as (
    SELECT
        ebook_months.ISBN13,
        ARRAY_AGG(STRUCT(
            ebook_months.release_date as month,
            crossref_events.metrics as crossref_events,
            google_analytics.metrics as google_analytics,
            google_books_sales.metrics as google_books_sales,
            google_books_traffic.metrics as google_books_traffic,
            jstor_country.metrics as jstor_country,
            jstor_institution.metrics as jstor_institution,
            irus_oapen.metrics as irus_oapen,
            irus_fulcrum.metrics as irus_fulcrum,
            ucl_discovery.metrics as ucl_discovery,
            worldreader.metrics as worldreader,
            internet_archive.metrics as internet_archive
        ) ORDER BY ebook_months.release_date DESC) as months
    FROM ebook_months
    LEFT JOIN google_analytics3_metrics as google_analytics ON ebook_months.ISBN13 = google_analytics.ISBN13 AND ebook_months.release_date = google_analytics.release_date
    LEFT JOIN google_books_sales_metrics as google_books_sales ON ebook_months.ISBN13 = google_books_sales.ISBN13 AND ebook_months.release_date = google_books_sales.release_date
    LEFT JOIN google_books_traffic_metrics as google_books_traffic ON ebook_months.ISBN13 = google_books_traffic.ISBN13 AND ebook_months.release_date = google_books_traffic.release_date
    LEFT JOIN jstor_country_metrics as jstor_country ON ebook_months.ISBN13 = jstor_country.ISBN13 AND ebook_months.release_date = jstor_country.release_date
    LEFT JOIN jstor_institution_metrics as jstor_institution ON ebook_months.ISBN13 = jstor_institution.ISBN13 AND ebook_months.release_date = jstor_institution.release_date
    LEFT JOIN irus_oapen_metrics as irus_oapen ON ebook_months.ISBN13 = irus_oapen.ISBN13 AND ebook_months.release_date = irus_oapen.release_date
    LEFT JOIN irus_fulcrum_metrics as irus_fulcrum ON ebook_months.ISBN13 = irus_fulcrum.ISBN13 AND ebook_months.release_date = irus_fulcrum.release_date
    LEFT JOIN ucl_discovery_metrics as ucl_discovery ON ebook_months.ISBN13 = ucl_discovery.ISBN13 AND ebook_months.release_date = ucl_discovery.release_date
    LEFT JOIN worldreader_metrics as worldreader ON ebook_months.ISBN13 = worldreader.ISBN13 AND ebook_months.release_date = worldreader.release_date
    LEFT JOIN internet_archive_metrics as internet_archive ON ebook_months.ISBN13 = internet_archive.ISBN13 AND ebook_months.release_date = internet_archive.release_date
    LEFT JOIN crossref_events as crossref_events ON ebook_months.ISBN13 = crossref_events.ISBN13 AND ebook_months.release_date = crossref_events.release_date
    WHERE google_analytics.metrics IS NOT NULL 
    OR google_books_sales.metrics IS NOT NULL 
    OR google_books_traffic.metrics IS NOT NULL 
    OR jstor_country.metrics IS NOT NULL 
    OR jstor_institution.metrics IS NOT NULL
    OR irus_oapen.metrics IS NOT NULL 
    OR irus_fulcrum.metrics IS NOT NULL 
    OR ucl_discovery.metrics IS NOT NULL
    OR worldreader.metrics IS NOT NULL
    OR internet_archive.metrics IS NOT NULL
    OR crossref_events.metrics IS NOT NULL 
    GROUP BY ebook_months.ISBN13
)

# Main Query
{#
The purpose of this query is to bring together a range of individual sources as compose them into the final query output.

From onix_ebook_titles, we get the list of books and from 'metrics' we we are able join all the metrics that were organised in the query above.
Then we are able to pull in specific metadata from across all the various sources, as seen in the sequence of LEFT JOINs below
#}
SELECT
    onix_ebook_titles.*,
    empty_work_ids.work_id,
    empty_work_family_ids.work_family_id,
    STRUCT(
        crossref_objects, chapters, events.overall as events, google_books_sales_metadata as google_books_sales,
        google_books_traffic_metadata as google_books_traffic, jstor_country_metadata as jstor_metadata,
        jstor_institution_metadata as jstor_institution_metadata, irus_oapen_metadata as irus_oapen_metadata,
        irus_fulcrum_metadata as irus_fulcrum_metadata, ucl_discovery_metadata as ucl_discovery_metadata, 
        worldreader_metadata as worldreader_metadata, internet_archive_metadata as internet_archive_metadata
    ) as metadata,
    metrics.months
FROM onix_ebook_titles
LEFT JOIN {% if onix_workflow %} `{{ workid_table_id  }}` {% else %} empty_work_ids {% endif %} as empty_work_ids on empty_work_ids.isbn13 = onix_ebook_titles.isbn13
LEFT JOIN {% if onix_workflow %} `{{ workfamilyid_table_id }}` {% else %} empty_work_family_ids {% endif %} as empty_work_family_ids on empty_work_family_ids.isbn13 = onix_ebook_titles.isbn13
LEFT JOIN metrics as metrics on metrics.ISBN13 = onix_ebook_titles.ISBN13
LEFT JOIN google_books_sales_metadata on google_books_sales_metadata.ISBN13  = onix_ebook_titles.ISBN13
LEFT JOIN google_books_traffic_metadata on google_books_traffic_metadata.ISBN13  = onix_ebook_titles.ISBN13
LEFT JOIN jstor_country_metadata on jstor_country_metadata.ISBN13 = onix_ebook_titles.ISBN13
LEFT JOIN jstor_institution_metadata on jstor_institution_metadata.ISBN13 = onix_ebook_titles.ISBN13
LEFT JOIN irus_oapen_metadata on irus_oapen_metadata.ISBN13 = onix_ebook_titles.ISBN13
LEFT JOIN irus_fulcrum_metadata on irus_fulcrum_metadata.ISBN13 = onix_ebook_titles.ISBN13
LEFT JOIN ucl_discovery_metadata on ucl_discovery_metadata.ISBN13 = onix_ebook_titles.ISBN13
LEFT JOIN worldreader_metadata on worldreader_metadata.ISBN13 = onix_ebook_titles.ISBN13
LEFT JOIN internet_archive_metadata on internet_archive_metadata.ISBN13 = onix_ebook_titles.ISBN13
LEFT JOIN `{{ book_table_id }}` as public_data on public_data.isbn = onix_ebook_titles.ISBN13;
