# AUTOGENERATED! DO NOT EDIT! File to edit: nbs/01_csv_bq.ipynb (unless otherwise specified).

__all__ = ['ssb_to_gcs', 'parquet_to_bigquery']

# Cell
import os
import re
import dask.dataframe as dd
from google.cloud import bigquery
from google.oauth2 import service_account
from nbdev.showdoc import *
from .core import clean_column_names

# Cell
def ssb_to_gcs(file: str):
    source = f'https://data.ssb.no/api/v0/dataset/{file}.csv'
    destination = f'gs://datafabrikken/ssb_datasett_{file}.parquet'
    df = dd.read_csv(source, encoding='latin-1',sep=';', blocksize=None).head(10)
    df.columns = [clean_column_names(x) for x in df.columns]
    df.to_parquet(destination, storage_options={'token': TOKEN})
    return destination

# Cell
def parquet_to_bigquery(file,table):
    credentials = service_account.Credentials.from_service_account_file(TOKEN)
    client = bigquery.Client(credentials=credentials, project=PROJECT_NAME)
    job_config = bigquery.LoadJobConfig()
    job_config.source_format = bigquery.SourceFormat.PARQUET
    job = client.load_table_from_uri(file,table,job_config=job_config)

    return job.result().state