{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Fetching checkpoint: 1.00kit [00:00, 315kit/s]                                                      \n",
      "Fetching encoder.json: 1.04Mit [00:00, 25.6Mit/s]                                                   \n",
      "Fetching hparams.json: 1.00kit [00:00, 429kit/s]                                                    \n",
      "Fetching model.ckpt.data-00000-of-00001: 1.42Git [00:57, 24.5Mit/s]                                 \n",
      "Fetching model.ckpt.index: 11.0kit [00:00, 2.87Mit/s]                                               \n",
      "Fetching model.ckpt.meta: 927kit [00:00, 28.0Mit/s]                                                 \n",
      "Fetching vocab.bpe: 457kit [00:00, 24.9Mit/s]                                                       \n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import sys\n",
    "import requests\n",
    "from tqdm import tqdm\n",
    "\n",
    "model = '345M'\n",
    "\n",
    "subdir = os.path.join('models', model)\n",
    "if not os.path.exists(subdir):\n",
    "    os.makedirs(subdir)\n",
    "subdir = subdir.replace('\\\\','/') # needed for Windows\n",
    "\n",
    "for filename in ['checkpoint','encoder.json','hparams.json','model.ckpt.data-00000-of-00001', 'model.ckpt.index', 'model.ckpt.meta', 'vocab.bpe']:\n",
    "\n",
    "    r = requests.get(\"https://storage.googleapis.com/gpt-2/\" + subdir + \"/\" + filename, stream=True)\n",
    "\n",
    "    with open(os.path.join(subdir, filename), 'wb') as f:\n",
    "        file_size = int(r.headers[\"content-length\"])\n",
    "        chunk_size = 1000\n",
    "        with tqdm(ncols=100, desc=\"Fetching \" + filename, total=file_size, unit_scale=True) as pbar:\n",
    "            # 1k for chunk_size, since Ethernet packet size is around 1500 bytes\n",
    "            for chunk in r.iter_content(chunk_size=chunk_size):\n",
    "                f.write(chunk)\n",
    "                pbar.update(chunk_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from google.cloud import storage\n",
    "client = storage.Client()\n",
    "bucket = client.bucket('mesolitica-general')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 7/7 [00:13<00:00,  1.90s/it]\n"
     ]
    }
   ],
   "source": [
    "from glob import glob\n",
    "from tqdm import tqdm\n",
    "\n",
    "files = glob('models/345M/*')\n",
    "for file in tqdm(files):\n",
    "    f = file.split('models/345M/')[1]\n",
    "    blob = bucket.blob(f'gpt2-345M-pretrained/{f}')\n",
    "    blob.upload_from_filename(file)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
