{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: bad escape \\s\n"
     ]
    }
   ],
   "source": [
    "from brightway2 import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "projects.set_current(\"CMLCA import\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Biosphere database already present!!! No setup is needed\n",
      "This database is already searchable\n"
     ]
    }
   ],
   "source": [
    "bw2setup()\n",
    "\n",
    "db = Database(\"biosphere3\")\n",
    "db.make_searchable()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "if \"ecoinvent 2.2\" not in databases:\n",
    "    ei = SingleOutputEcospold1Importer(\n",
    "        \"/Users/cmutel/Documents/LCA Documents/Ecoinvent/2.2/processes\", \n",
    "        \"ecoinvent 2.2\"\n",
    "    )\n",
    "    ei.apply_strategies()\n",
    "    ei.write_database()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracted 15 worksheets in 0.04 seconds\n"
     ]
    }
   ],
   "source": [
    "fp = (\"Life cycle inventory of the production of rare earths and \"\n",
    "      \"the subsequent production of NdFeB rare earth permanent \"\n",
    "      \"magnets - supporting tables.xlsx\")\n",
    "\n",
    "ei = ExcelImporter(fp, \"Rare earth metals\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def reformat_sheet(sheet, ds, database_name):\n",
    "    if ds[0][0][:10] != \"Process = \":\n",
    "        print(\"Ignoring worksheet {} - not valid inventory\".format(sheet))\n",
    "        return\n",
    "\n",
    "    metadata = [\n",
    "        # (\"Process = \", 'name'),\n",
    "        ('Description = ', 'description'),\n",
    "        ('ID = ', 'cmlca id'),\n",
    "        ('Stage = ', 'stage'),\n",
    "        ('Region = ', 'location'),\n",
    "        ('Author = ', 'author'),\n",
    "        ('Date = ', 'date'),\n",
    "    ]\n",
    "    biosphere_flows = {\n",
    "        \"Environmental resources\", \n",
    "        \"Environmental emissions\"\n",
    "    }\n",
    "\n",
    "    data, obj = [], None\n",
    "\n",
    "    for index in range(len(ds)):\n",
    "        if ds[index][0][:10] == \"Process = \":\n",
    "            if obj:\n",
    "                data.append(obj)\n",
    "            obj = {\n",
    "                'sheet': sheet,\n",
    "                'database': database_name,\n",
    "                'exchanges': [],\n",
    "                'name': ds[index][0].replace(\"Process = \", '')\n",
    "            }\n",
    "        elif any(ds[index][0][:len(x)] == x for x, y in metadata):\n",
    "            for label, value in metadata:\n",
    "                if ds[index][0][:len(label)] == label:\n",
    "                    obj[value] = ds[index][0].replace(label, '')\n",
    "        else:\n",
    "            row = ds[index]\n",
    "            if row[0] == 'Label' or not row[0]:\n",
    "                continue\n",
    "            elif row[0] in biosphere_flows:\n",
    "                kind = 'biosphere'\n",
    "                continue\n",
    "            elif row[0] == \"Economic inflows\":\n",
    "                kind = 'technosphere'\n",
    "                continue\n",
    "            elif row[0] == \"Economic outflows\":\n",
    "                kind = 'production'\n",
    "                continue\n",
    "            elif not all(row):\n",
    "                continue\n",
    "            obj['exchanges'].append({\n",
    "                'code': row[0],\n",
    "                'name': row[1],\n",
    "                'amount': row[2],\n",
    "                'unit': row[3],\n",
    "                'uncertainty': row[4],\n",
    "                'type': kind,\n",
    "            })\n",
    "    if obj:\n",
    "        data.append(obj)\n",
    "    return data\n",
    "\n",
    "\n",
    "def cmlca_reformat(data, database_name):\n",
    "    \"\"\"Change from extracted Excel cells to standard BW2 format\"\"\"\n",
    "    return [\n",
    "        x\n",
    "        for row in data\n",
    "        for x in reformat_sheet(sheet=row[0], ds=row[1], database_name=database_name) \n",
    "    ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def reformat_biosphere_names(data):\n",
    "    \"\"\"Change from:\n",
    "\n",
    "        \"name\": \"Water, unspecified natural origin[resource_in water]\" \n",
    "\n",
    "    to: \n",
    "\n",
    "        \"name\": \"Water, unspecified natural origin\",\n",
    "        \"categories\": (\"resource\", \"in water\")\n",
    "\n",
    "    \"\"\"\n",
    "    for ds in data:\n",
    "        for exc in  ds.get('exchanges', []):\n",
    "            if exc['type'] == \"biosphere\":\n",
    "                # Use simple text processing instead of overly clever regex\n",
    "                split_string = exc['name'].split(\"[\")\n",
    "                exc['name'] = \"[\".join(split_string[:-1])\n",
    "                exc['categories'] = tuple(\n",
    "                    split_string[-1].replace(\"]\", \"\").split(\"_\")\n",
    "                )\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def reformat_technosphere_names(data):\n",
    "    \"\"\"Change from:\n",
    "\n",
    "        \"name\": \"electricity, medium voltage, at grid[CN]\" \n",
    "\n",
    "    to: \n",
    "\n",
    "        \"name\": \"electricity, medium voltage, at grid\",\n",
    "        \"location\": \"CN\"\n",
    "\n",
    "    \"\"\"\n",
    "    for ds in data:\n",
    "        for exc in  ds.get('exchanges', []):\n",
    "            if exc['type'] == \"technosphere\" and (\"[\" in exc['name'] and \"]\" in exc['name']):\n",
    "                # Use simple text processing instead of overly clever regex\n",
    "                split_string = exc['name'].split(\"[\")\n",
    "                exc['name'] = \"[\".join(split_string[:-1])\n",
    "                exc['location'] = split_string[-1].replace(\"]\", \"\")\n",
    "                \n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def set_production_code_and_amount(data):\n",
    "    \"\"\"Set dataset production amount and code from single production exchange\"\"\"\n",
    "    for ds in data:\n",
    "        products = [x for x in ds.get('exchanges', []) if x.get('type') == 'production']\n",
    "        if len(products) == 1:\n",
    "            product = products[0]\n",
    "            ds['code'] = product['code']\n",
    "            ds['production amount'] = product['amount']\n",
    "        else:\n",
    "            print(\"Can't process multioutput process: {}: {}\".format(ds['sheet'], ds['name']))\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "cmlca_migration = {\n",
    "    'fields': ['name', 'categories', 'type'],\n",
    "    'data': [\n",
    "        (\n",
    "            ('Water', ('resources',), 'biosphere'),\n",
    "            {\n",
    "                'categories': ('water',)\n",
    "            }\n",
    "        ),\n",
    "        (\n",
    "            ('Carbon dioxide', ('air',), 'biosphere'),\n",
    "            {\n",
    "                'name': 'Carbon dioxide, fossil',\n",
    "            }\n",
    "        ),\n",
    "        (\n",
    "            ('Carbon monoxide', ('air',), 'biosphere'),\n",
    "            {\n",
    "                'name': 'Carbon monoxide, fossil',\n",
    "            }\n",
    "        ),\n",
    "        (\n",
    "            ('2,3,7,8-tetrachlorodibenzo-p-dioxin', ('air',), 'biosphere'),\n",
    "            {\n",
    "                'name': 'Dioxins, measured as 2,3,7,8-tetrachlorodibenzo-p-dioxin',\n",
    "            }\n",
    "        ),\n",
    "        (\n",
    "            ('Particulates', ('air',), 'biosphere'),\n",
    "            {\n",
    "                'name': 'Particulates, > 2.5 um, and < 10um',\n",
    "            }\n",
    "        ),\n",
    "        (\n",
    "            ('Water, unspecified natural origin', ['natural resource', 'in water'], 'biosphere'),\n",
    "            {\n",
    "                'unit': 'cubic meter',\n",
    "            }\n",
    "        ),\n",
    "    ]\n",
    "}\n",
    "\n",
    "cmlca_migration_units = {\n",
    "    'fields': ['unit'],\n",
    "    'data': [\n",
    "        (\n",
    "            ('m2/m2',),\n",
    "            {'unit': \"square meter\"}\n",
    "        ),\n",
    "        (\n",
    "            ('kg/m2',),\n",
    "            {'unit': \"kilogram\"}\n",
    "        ),\n",
    "        (\n",
    "            ('m3/m2',),\n",
    "            {'unit': \"cubic meter\"}\n",
    "        ),\n",
    "        (\n",
    "            ('kWh/m2',),\n",
    "            {'unit': \"kilowatt hour\"}\n",
    "        ),\n",
    "    ]\n",
    "}\n",
    "\n",
    "if \"cmlca\" not in migrations:\n",
    "    Migration(\"cmlca\").write(\n",
    "        cmlca_migration, \n",
    "        \"Reformat biosphere flows to ecoinvent standards\"\n",
    "    )\n",
    "    Migration(\"cmlca units\").write(\n",
    "        cmlca_migration_units, \n",
    "        \"Reformat electroplating flows to correct functional unit\"\n",
    "    )    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from bw2io.strategies import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Applying strategy: cmlca_reformat\n",
      "Applying strategy: reformat_biosphere_names\n",
      "Applying strategy: reformat_technosphere_names\n",
      "Applying strategy: set_production_code_and_amount\n",
      "Can't process multioutput process: Beneficiation: [P4098] beneficiation of REE ore, baseline\n",
      "Can't process multioutput process: Acid roasting: [P4097] REE acid roasting, baseline\n",
      "Can't process multioutput process: Solvent extraction: [P4096] REO solvent extraction, baseline\n",
      "Can't process multioutput process: Grinding&slicing: [P4093] grinding and slicing of NdFeB, primary, baseline\n",
      "Applying strategy: drop_unspecified_subcategories\n",
      "Applying strategy: normalize_biosphere_categories\n",
      "Applying strategy: normalize_biosphere_names\n",
      "Applying strategy: normalize_units\n",
      "Applying strategy: migrate_exchanges\n",
      "Applying strategy: migrate_exchanges\n",
      "Applying strategy: migrate_exchanges\n",
      "Applying strategy: link_technosphere_by_activity_hash\n",
      "Couldn't apply strategy link_technosphere_by_activity_hash:\n",
      "\tNot all datasets in database to be linked have ``database`` or ``code`` attributes\n",
      "Applied 12 strategies in 0.02 seconds\n"
     ]
    }
   ],
   "source": [
    "import functools\n",
    "\n",
    "strategies = [\n",
    "    functools.partial(cmlca_reformat, database_name=ei.db_name),\n",
    "    reformat_biosphere_names,\n",
    "    reformat_technosphere_names,\n",
    "    set_production_code_and_amount,\n",
    "    drop_unspecified_subcategories,\n",
    "    normalize_biosphere_categories,\n",
    "    normalize_biosphere_names,\n",
    "    normalize_units,\n",
    "    functools.partial(migrate_exchanges, migration=\"cmlca\"),\n",
    "    functools.partial(migrate_exchanges, migration=\"cmlca units\"),\n",
    "    functools.partial(migrate_exchanges, migration=\"biosphere-2-3-names\"),\n",
    "    link_technosphere_by_activity_hash,\n",
    "]\n",
    "\n",
    "ei.apply_strategies(strategies)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Applying strategy: link_iterable_by_fields\n",
      "Applying strategy: link_iterable_by_fields\n"
     ]
    }
   ],
   "source": [
    "ei.match_database(\"biosphere3\", kind=\"biosphere\")\n",
    "ei.match_database(\"ecoinvent 2.2\", fields=['name', 'location', 'unit'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "16 datasets\n",
      "229 exchanges\n",
      "33 unlinked exchanges\n",
      "  Type production: 18 unique unlinked exchanges\n",
      "  Type technosphere: 15 unique unlinked exchanges\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(16, 229, 33)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ei.statistics()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wrote matching file to:\n",
      "/Users/cmutel/Library/Application Support/Brightway3/CMLCA-import.ec6c7b98f76633079e98132f49507f1e/export/db-matching-Rare-earth-metals.xlsx\n"
     ]
    }
   ],
   "source": [
    "ei.write_excel()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
