{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-02T13:57:23.713755Z",
     "start_time": "2019-01-02T13:57:21.059562Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/xiaofeiwu/miniconda3/envs/bigdata/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "INFO:sagas.provider.hanlp_utils:Trying to connect hanlp rpc with address 'localhost:10052'\n"
     ]
    }
   ],
   "source": [
    "import logging\n",
    "import os\n",
    "\n",
    "import pytest\n",
    "from rasa_nlu import data_router, config\n",
    "from rasa_nlu.components import ComponentBuilder\n",
    "from rasa_nlu.model import Trainer\n",
    "from rasa_nlu.utils import zip_folder\n",
    "from rasa_nlu import training_data\n",
    "\n",
    "from sagas.provider.hanlp_utils import Hanlp\n",
    "\n",
    "# logging.basicConfig(level=\"DEBUG\")\n",
    "logging.basicConfig(level=\"INFO\")\n",
    "CONFIG_DEFAULTS_PATH = \"sample_configs/config_defaults.yml\"\n",
    "DEFAULT_DATA_PATH = \"data/examples/rasa/demo-rasa.json\"\n",
    "TEST_MODEL_PATH = \"test_models/test_model_spacy_sklearn\"\n",
    "\n",
    "def component_builder():\n",
    "    return ComponentBuilder()\n",
    "def hanlp(component_builder, default_config):\n",
    "    return component_builder.create_component(\"sagas.provider.hanlp_utils.Hanlp\", default_config)\n",
    "def timenlp(component_builder, default_config):\n",
    "    return component_builder.create_component(\"sagas.provider.time_extractor.TimeExtractor\", default_config)\n",
    "\n",
    "def default_config():\n",
    "    return config.load(CONFIG_DEFAULTS_PATH)\n",
    "\n",
    "# component_classes = [Hanlp]\n",
    "# registered_components = {c.name: c for c in component_classes}\n",
    "hanlp=hanlp(component_builder(), default_config())\n",
    "timenlp=timenlp(component_builder(), default_config())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-02T13:36:40.276264Z",
     "start_time": "2019-01-02T13:36:40.258263Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<module 'sagas.provider.hanlp_entity_extractor' from '/Users/xiaofeiwu/jcloud/assets/langs/workspace/rasa/stack/sagas/provider/hanlp_entity_extractor.py'>"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import sagas.provider.hanlp_entity_extractor\n",
    "import imp\n",
    "imp.reload(sagas.provider.hanlp_entity_extractor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-02T13:53:04.354053Z",
     "start_time": "2019-01-02T13:53:04.298407Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total entities 1\n",
      "{'entity': 'person', 'value': '张晚霞', 'start': 7, 'confidence': None, 'end': 10, 'extractor': 'ner_hanlp'}\n",
      "total entities 6\n",
      "{'entity': 'person', 'value': '蓝翔', 'start': 0, 'confidence': None, 'end': 2, 'extractor': 'ner_hanlp'}\n",
      "{'entity': 'location', 'value': '宁夏', 'start': 3, 'confidence': None, 'end': 5, 'extractor': 'ner_hanlp'}\n",
      "{'entity': 'location', 'value': '固原市', 'start': 5, 'confidence': None, 'end': 8, 'extractor': 'ner_hanlp'}\n",
      "{'entity': 'location', 'value': '彭阳县', 'start': 8, 'confidence': None, 'end': 11, 'extractor': 'ner_hanlp'}\n",
      "{'entity': 'location', 'value': '红河镇', 'start': 11, 'confidence': None, 'end': 14, 'extractor': 'ner_hanlp'}\n",
      "{'entity': 'location', 'value': '黑牛沟村', 'start': 14, 'confidence': None, 'end': 18, 'extractor': 'ner_hanlp'}\n"
     ]
    }
   ],
   "source": [
    "from rasa_nlu.config import RasaNLUModelConfig\n",
    "from rasa_nlu.extractors.spacy_entity_extractor import SpacyEntityExtractor\n",
    "from rasa_nlu.training_data import TrainingData, Message\n",
    "from sagas.provider.hanlp_entity_extractor import HanlpEntityExtractor\n",
    "\n",
    "def test_hanlp_ner_extractor(text, hanlp, hanlp_doc):\n",
    "    ext = HanlpEntityExtractor()\n",
    "    \n",
    "    example = Message(text, {\n",
    "        \"intent\": \"wish\",\n",
    "        \"entities\": [],\n",
    "        \"hanlp_doc\": hanlp_doc})\n",
    "\n",
    "    ext.process(example, hanlp=hanlp.nlp)\n",
    "\n",
    "    print(\"total entities\", len(example.get(\"entities\", [])))\n",
    "    for ent in example.get(\"entities\"):\n",
    "        print(ent)\n",
    "\n",
    "text=\"我的希望是希望张晚霞的背影被晚霞映红\"\n",
    "test_hanlp_ner_extractor(text, hanlp, hanlp.doc_for_text(text))\n",
    "text=\"蓝翔给宁夏固原市彭阳县红河镇黑牛沟村捐赠了挖掘机\"\n",
    "test_hanlp_ner_extractor(text, hanlp, hanlp.doc_for_text(text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-02T13:53:55.337560Z",
     "start_time": "2019-01-02T13:53:55.221135Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total entities 1\n",
      "{'entity': 'amount', 'value': '19', 'start': 0, 'confidence': None, 'end': 2, 'extractor': 'ner_amount'}\n",
      "total entities 2\n",
      "{'entity': 'amount', 'value': '300', 'start': 2, 'confidence': None, 'end': 5, 'extractor': 'ner_amount'}\n",
      "{'entity': 'amount', 'value': '2', 'start': 7, 'confidence': None, 'end': 8, 'extractor': 'ner_amount'}\n"
     ]
    }
   ],
   "source": [
    "from sagas.provider.amount_extractor import AmountExtractor\n",
    "\n",
    "def test_amount_ner_extractor(text, hanlp, hanlp_doc):\n",
    "    ext = AmountExtractor()\n",
    "    \n",
    "    example = Message(text, {\n",
    "        \"intent\": \"wish\",\n",
    "        \"entities\": [],\n",
    "        \"hanlp_doc\": hanlp_doc})\n",
    "\n",
    "    ext.process(example, hanlp=hanlp.nlp)\n",
    "\n",
    "    print(\"total entities\", len(example.get(\"entities\", [])))\n",
    "    for ent in example.get(\"entities\"):\n",
    "        print(ent)\n",
    "\n",
    "text=\"十九元套餐包括什么\"\n",
    "test_amount_ner_extractor(text, hanlp, hanlp.doc_for_text(text))\n",
    "text=\"牛奶三〇〇克*2\"\n",
    "test_amount_ner_extractor(text, hanlp, hanlp.doc_for_text(text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-02T13:48:59.254244Z",
     "start_time": "2019-01-02T13:48:59.227004Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "我 0\n",
      "想 1\n",
      "去 2\n",
      "吃 3\n",
      "兰州 4\n",
      "拉面 6\n"
     ]
    }
   ],
   "source": [
    "from sagas.provider.hanlp_tokenizer import HanlpTokenizer\n",
    "def test_hanlp_tokenizer(text, hanlp, hanlp_doc):\n",
    "    ext = HanlpTokenizer()\n",
    "    \n",
    "    example = Message(text, {\n",
    "        \"intent\": \"wish\",\n",
    "        \"entities\": [],\n",
    "        \"hanlp_doc\": hanlp_doc})\n",
    "\n",
    "    ext.process(example, hanlp=hanlp)\n",
    "    for token in example.get(\"tokens\"):\n",
    "        print(token.text, token.offset)\n",
    "\n",
    "# text=\"我的希望是希望张晚霞的背影被晚霞映红\"\n",
    "text=\"我想去吃兰州拉面\"\n",
    "test_hanlp_tokenizer(text, hanlp, hanlp.doc_for_text(text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-02T14:27:35.342611Z",
     "start_time": "2019-01-02T14:27:35.333474Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<module 'sagas.provider.time_extractor' from '/Users/xiaofeiwu/jcloud/assets/langs/workspace/rasa/stack/sagas/provider/time_extractor.py'>"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import sagas.provider.time_extractor\n",
    "import imp\n",
    "imp.reload(sagas.provider.time_extractor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-02T14:33:38.331837Z",
     "start_time": "2019-01-02T14:33:38.164890Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:time-nlp gateway host/port: 127.0.0.1 25333\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total entities 2\n",
      "total entities 2\n",
      "{'entity': 'time', 'value': '2019-01-06 00:00:00', 'start': 0, 'confidence': None, 'end': 3, 'additional_info': 'True', 'extractor': 'sagas.provider.time_extractor.TimeExtractor'}\n",
      "{'entity': 'time', 'value': '2019-01-13 00:00:00', 'start': 4, 'confidence': None, 'end': 7, 'additional_info': 'True', 'extractor': 'sagas.provider.time_extractor.TimeExtractor'}\n"
     ]
    }
   ],
   "source": [
    "from rasa_nlu.config import RasaNLUModelConfig\n",
    "from rasa_nlu.training_data import TrainingData, Message\n",
    "\n",
    "CONFIG_ZH_PATH = \"sample_configs/config_zh.yml\"\n",
    "def test_time_entity_extractor(component_builder):\n",
    "    # _config = RasaNLUModelConfig({\"pipeline\": [{\"name\": \"sagas.provider.time_extractor.TimeExtractor\"}]})\n",
    "    # _config.set_component_attr(\"ner_time\", dimensions=[\"time\"], host=\"unknown\")\n",
    "    _config=config.load(CONFIG_ZH_PATH)\n",
    "    c = component_builder.create_component(\"sagas.provider.time_extractor.TimeExtractor\", _config)\n",
    "    message = Message(\"周五下午7点到8点\")\n",
    "    c.process(message)\n",
    "    entities = message.get(\"entities\")\n",
    "    print(\"total entities\", len(entities))\n",
    "\n",
    "    # Test with a defined date\n",
    "    # 1381536182000 == 2013/10/12 02:03:02\n",
    "    message = Message(\"本周日到下周日出差\", time=\"1381536182000\")\n",
    "    c.process(message)\n",
    "    entities = message.get(\"entities\")\n",
    "    print(\"total entities\", len(entities))\n",
    "    for ent in entities:\n",
    "        print(ent)\n",
    "\n",
    "test_time_entity_extractor(component_builder())"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
