{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.insert(0, '..')\n",
    "\n",
    "from formasaurus import formtype_features as features\n",
    "from formasaurus import formtype_model\n",
    "from formasaurus.storage import Storage, load_html"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Available training data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Checking: 100%|##########################| 832/832 [00:03<00:00, 269.34 files/s]\n",
      "                                          "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Status: OK\n",
      "Annotated HTML forms (simplified classes):\n",
      "\n",
      "364   search                    (s)\n",
      "222   login                     (l)\n",
      "153   registration              (r)\n",
      "122   other                     (o)\n",
      "120   contact/comment           (c)\n",
      "107   join mailing list         (m)\n",
      "95    password/login recovery   (p)\n",
      "62    order/add to cart         (b)\n",
      "\n",
      "Total form count: 1245\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    }
   ],
   "source": [
    "storage = Storage(\"../formasaurus/data\")\n",
    "storage.check()\n",
    "storage.print_form_type_counts(simplify=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load training / evaluation data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading: 832 files [00:03, 223.44 files/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "annotations = list(storage.iter_annotations(\n",
    "    simplify_form_types=True,\n",
    "    simplify_field_types=True,\n",
    "    verbose=True,\n",
    "    leave=True,        \n",
    "))\n",
    "X, y = formtype_model.get_Xy(annotations, full_type_names=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Ideas for useful features\n",
    "\n",
    "### Search forms\n",
    "\n",
    "* a single query field\n",
    "* a field named \"q\" or \"s\"\n",
    "* \"search\" in URL\n",
    "* \"search\" in submit button text (submit value)\n",
    "* \"search\" in form css class or id\n",
    "* no password field\n",
    "* method == GET?\n",
    "\n",
    "### Login forms\n",
    "\n",
    "* username or email and password\n",
    "* 2 passwords - likely not a login form\n",
    "* \"login\" or \"sign in\" (or variations) in URL\n",
    "* \"login\" or \"sign in\" (or variations) in form css class or id\n",
    "* \"login\" or \"sign in\" in submit button text\n",
    "* \"Remember me\" checkbox (or any single checkbox)\n",
    "* no select elements\n",
    "* no textarea elements\n",
    "* openid?\n",
    "* method == POST\n",
    "\n",
    "### Registration forms\n",
    "\n",
    "* 2 passwords \n",
    "* \"register\" / \"sign up\" in URL, form css class / id or submit button text\n",
    "* \"agree\" checkbox\n",
    "* email\n",
    "* username\n",
    "* method == POST\n",
    "\n",
    "### Contact forms\n",
    "\n",
    "* feedback in URL/class\n",
    "* textarea\n",
    "* \"Send\" button\n",
    "* email\n",
    "* method == POST\n",
    "\n",
    "### Password reset\n",
    "\n",
    "* a single email or username field\n",
    "* \"password\" in URL/css class/ submit button text\n",
    "\n",
    "### Join Mailing List\n",
    "\n",
    "* a single email field\n",
    "* subscribe/join/newsletter words\n",
    "* a short form\n",
    "\n",
    "The main problem with \"join mailing list\" forms is to distinguish them from search forms.\n",
    "\n",
    "## How to handle them\n",
    "\n",
    "Instead of hardcoding the features above many of them are generalized. For exmaple, instead of writing \"search in URL\" we extract all 5-character substrings from the URL and use \"`urlsubstring<N>` in URL\" as features. There are some disadvantages in this approach, but it provides a good starting point.\n",
    "\n",
    "The feature extractors are stored in formtype.features module."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn.feature_extraction import DictVectorizer\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n",
    "from sklearn.pipeline import Pipeline, FeatureUnion, make_pipeline, make_union\n",
    "from sklearn.linear_model import LogisticRegression, SGDClassifier\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.metrics import classification_report, precision_recall_fscore_support"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                         precision    recall  f1-score   support\n",
      "\n",
      "                 search       0.91      0.96      0.93       364\n",
      "                  login       0.96      0.96      0.96       222\n",
      "           registration       0.96      0.85      0.90       153\n",
      "password/login recovery       0.87      0.86      0.87        95\n",
      "        contact/comment       0.85      0.94      0.89       120\n",
      "      join mailing list       0.89      0.88      0.88       107\n",
      "      order/add to cart       0.93      0.66      0.77        62\n",
      "                  other       0.67      0.70      0.69       122\n",
      "\n",
      "            avg / total       0.89      0.89      0.89      1245\n",
      "\n",
      "89.0% forms are classified correctly.\n",
      "CPU times: user 12.6 s, sys: 63 ms, total: 12.7 s\n",
      "Wall time: 12.7 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# reload(features)\n",
    "from formasaurus.formtype_model import _create_feature_union\n",
    "# ======= define the model ========\n",
    "\n",
    "# features should be kept in sync with formasaurus.formtype_features\n",
    "# a list of 3-tuples with default features:\n",
    "# (feature_name, form_transformer, vectorizer)\n",
    "FEATURES = [    \n",
    "    (\n",
    "        # bias feature is for easier debugging, it should be removed\n",
    "        # in production!\n",
    "        \"bias\",  \n",
    "        features.Bias(),\n",
    "        DictVectorizer(),  \n",
    "    ),\n",
    "    \n",
    "    (\n",
    "        \"form elements\",\n",
    "        features.FormElements(),\n",
    "        DictVectorizer()\n",
    "    ),\n",
    "    (\n",
    "        \"<input type=submit value=...>\",\n",
    "        features.SubmitText(),\n",
    "        CountVectorizer(ngram_range=(1,2), min_df=1, binary=True)\n",
    "    ),\n",
    "    (\n",
    "        \"<a> TEXT </a>\",\n",
    "        features.FormLinksText(),\n",
    "        TfidfVectorizer(ngram_range=(1,2), min_df=4, binary=True,\n",
    "                        stop_words={'and', 'or', 'of'})\n",
    "    ),\n",
    "    (\n",
    "        \"<label> TEXT </label>\",\n",
    "        features.FormLabelText(),\n",
    "        TfidfVectorizer(ngram_range=(1,2), min_df=3, binary=True,\n",
    "                        stop_words=\"english\")\n",
    "    ),\n",
    "\n",
    "    (\n",
    "        \"<form action=...>\",\n",
    "        features.FormUrl(),\n",
    "        TfidfVectorizer(ngram_range=(5,6), min_df=4, binary=True,\n",
    "                        analyzer=\"char_wb\")\n",
    "    ),\n",
    "    (\n",
    "        \"<form class=... id=...>\",\n",
    "        features.FormCss(),\n",
    "        TfidfVectorizer(ngram_range=(4,5), min_df=3, binary=True,\n",
    "                        analyzer=\"char_wb\")\n",
    "    ),\n",
    "    (\n",
    "        \"<input class=... id=...>\",\n",
    "        features.FormInputCss(),\n",
    "        TfidfVectorizer(ngram_range=(4,5), min_df=5, binary=True,\n",
    "                        analyzer=\"char_wb\")\n",
    "    ),\n",
    "    (\n",
    "        \"<input name=...>\",\n",
    "        features.FormInputNames(),\n",
    "        TfidfVectorizer(ngram_range=(5,6), min_df=3, binary=True,\n",
    "                        analyzer=\"char_wb\")\n",
    "    ),\n",
    "    (\n",
    "        \"<input title=...>\",\n",
    "        features.FormInputTitle(),\n",
    "        TfidfVectorizer(ngram_range=(5,6), min_df=3, binary=True,\n",
    "                        analyzer=\"char_wb\")\n",
    "    ),\n",
    "]\n",
    "\n",
    "\n",
    "# clf = SGDClassifier(\n",
    "#     penalty='elasticnet', \n",
    "#     loss='log', \n",
    "#     alpha=0.0002,\n",
    "#     fit_intercept=False, \n",
    "#     shuffle=True, \n",
    "#     random_state=0,\n",
    "#     n_iter=50,\n",
    "# )\n",
    "clf = LogisticRegression(penalty='l2', C=5, fit_intercept=False, random_state=0, tol=0.01)\n",
    "\n",
    "# clf = LinearSVC(C=0.5, random_state=0, fit_intercept=False)\n",
    "model = Pipeline([\n",
    "    ('fe', _create_feature_union(FEATURES)),\n",
    "    ('clf', clf),\n",
    "])\n",
    "\n",
    "formtype_model.print_classification_report(annotations, n_folds=10, model=model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Check what classifier learned\n",
    "\n",
    "For linear classifiers like Logistic Regression or an SVM without a kernel we can check coefficient values to understand better how the decision is made. \n",
    "\n",
    "For correlated features (notably, n-grams) weight will be spread across all correlated features, so just checking coefficients is not enough, but looking at them is useful anyways."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_informative_features(vectorizers, clf, class_labels, N):\n",
    "    \"\"\"\n",
    "    Return text with features with the highest absolute coefficient\n",
    "    values, per class.\n",
    "    \"\"\"\n",
    "    feature_names = []\n",
    "    for vec_name, vec in vectorizers:\n",
    "        feature_names.extend(\n",
    "            \"%30s  %s\" % (vec_name, name) for name in vec.get_feature_names()\n",
    "        )\n",
    "    features_by_class = []\n",
    "    for i, class_label in enumerate(class_labels):\n",
    "        topN = np.argsort(clf.coef_[i])[-N:]\n",
    "        bottomN = np.argsort(clf.coef_[i])[:N]\n",
    "        res = []\n",
    "\n",
    "        for j in reversed(topN):\n",
    "            coef = clf.coef_[i][j]\n",
    "            if coef > 0:\n",
    "                res.append(\"+%0.4f: %s\" % (coef, feature_names[j]))\n",
    "\n",
    "        if (len(topN) >= N) or (len(bottomN) >= N):\n",
    "            res.append(\"   ...\")\n",
    "\n",
    "        for j in reversed(bottomN):\n",
    "            coef = clf.coef_[i][j]\n",
    "            if coef < 0:\n",
    "                res.append(\"%0.4f: %s\" % (coef, feature_names[j]))\n",
    "        features_by_class.append((class_label, '\\n'.join(res)))\n",
    "    return features_by_class\n",
    "\n",
    "\n",
    "def print_informative_features(features, clf, top, classes=None, class_map=None):\n",
    "    vectorizers = [(name, vec) for (name, fe, vec) in features]\n",
    "    feat_info = get_informative_features(vectorizers, clf, clf.classes_, top)\n",
    "    for cls, report in feat_info:\n",
    "        if classes is not None and cls not in classes:\n",
    "            continue\n",
    "        if class_map is not None:\n",
    "            print(class_map[cls])\n",
    "        else:\n",
    "            print(cls)\n",
    "        print(report)\n",
    "        print(\"-\"*80)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "contact/comment\n",
      "+5.7047:                  form elements  has <textarea>\n",
      "+2.4165:          <label> TEXT </label>  question\n",
      "+1.6249:              <form action=...>   kont\n",
      "+1.5064:                  <a> TEXT </a>  agent\n",
      "+1.5023:                  <a> TEXT </a>  affiliate\n",
      "+1.4561:                  <a> TEXT </a>  mobile\n",
      "+1.4456:                  <a> TEXT </a>  forums\n",
      "+1.4300:                  <a> TEXT </a>  privacy statement\n",
      "+1.3853:                  <a> TEXT </a>  statement\n",
      "+1.2378:                  <a> TEXT </a>  contact\n",
      "+1.2010:          <label> TEXT </label>  message\n",
      "+1.1773:  <input type=submit value=...>  send\n",
      "+1.1518:                  <a> TEXT </a>  contact us\n",
      "+1.0653:                  <a> TEXT </a>  us\n",
      "+0.8713:                  <a> TEXT </a>  privacy\n",
      "+0.8337:        <form class=... id=...>  ques\n",
      "+0.8337:        <form class=... id=...>  quest\n",
      "+0.8337:        <form class=... id=...>  uest\n",
      "+0.7949:                  <a> TEXT </a>  http\n",
      "+0.7949:                  <a> TEXT </a>  http www\n",
      "+0.7931:          <label> TEXT </label>  comments\n",
      "+0.7820:        <form class=... id=...>  new_\n",
      "+0.7761:          <label> TEXT </label>  email message\n",
      "+0.7518:        <form class=... id=...>  t-for\n",
      "+0.7227:              <form action=...>   send\n",
      "   ...\n",
      "-0.6002:        <form class=... id=...>   web\n",
      "-0.6036:  <input type=submit value=...>  crear\n",
      "-0.6093:        <form class=... id=...>   ord\n",
      "-0.6216:  <input type=submit value=...>  now\n",
      "-0.6263:              <form action=...>  .aspx\n",
      "-0.6448:        <form class=... id=...>  site\n",
      "-0.6522:  <input type=submit value=...>  next\n",
      "-0.6587:        <form class=... id=...>  ine \n",
      "-0.6949:               <input name=...>   mail \n",
      "-0.7081:          <label> TEXT </label>  login\n",
      "-0.7184:               <input name=...>   mail\n",
      "-0.7209:              <form action=...>  entry\n",
      "-0.7577:              <form action=...>  tiona\n",
      "-0.8362:  <input type=submit value=...>  registrovať\n",
      "-0.9821:  <input type=submit value=...>  корзину отправить\n",
      "-0.9821:  <input type=submit value=...>  корзину\n",
      "-1.0001:                  form elements  has <input type=checkbox>\n",
      "-1.0979:                  form elements  exactly one <input type=password>\n",
      "-1.1312:  <input type=submit value=...>  lähetä\n",
      "-1.1847:        <form class=... id=...>   add\n",
      "-1.3446:                  form elements  <form method=post\n",
      "-1.5250:                  form elements  <form method=get\n",
      "-1.7971:                  form elements  exactly one <input type=text>\n",
      "-2.4403:                  form elements  exactly two <input type=password>\n",
      "-2.8696:                           bias  bias\n",
      "--------------------------------------------------------------------------------\n",
      "join mailing list\n",
      "+1.6450:               <input name=...>   go \n",
      "+1.5434:  <input type=submit value=...>  subscribe\n",
      "+1.4717:  <input type=submit value=...>  подписаться\n",
      "+1.4145:        <form class=... id=...>  mail\n",
      "+1.4101:               <input name=...>   emai\n",
      "+1.4101:               <input name=...>   email\n",
      "+1.2601:               <input name=...>  news \n",
      "+1.1599:        <form class=... id=...>  news\n",
      "+1.1290:  <input type=submit value=...>  cadastrar\n",
      "+1.1240:               <input name=...>   zip \n",
      "+1.1069:               <input name=...>  email\n",
      "+1.0609:                  <a> TEXT </a>  list\n",
      "+1.0256:               <input name=...>   em \n",
      "+1.0011:  <input type=submit value=...>  up\n",
      "+0.9916:        <form class=... id=...>  ubsc\n",
      "+0.9916:        <form class=... id=...>  bscr\n",
      "+0.9916:        <form class=... id=...>  bscri\n",
      "+0.9916:        <form class=... id=...>  ubscr\n",
      "+0.9916:        <form class=... id=...>  subsc\n",
      "+0.9916:        <form class=... id=...>  subs\n",
      "+0.9785:        <form class=... id=...>  scri\n",
      "+0.9651:                  form elements  has <input type=email>\n",
      "+0.9158:        <form class=... id=...>  lett\n",
      "+0.9158:        <form class=... id=...>  lette\n",
      "+0.9158:        <form class=... id=...>  ewsle\n",
      "   ...\n",
      "-0.5800:        <form class=... id=...>  earch\n",
      "-0.5800:        <form class=... id=...>  searc\n",
      "-0.5800:        <form class=... id=...>  arch\n",
      "-0.5800:        <form class=... id=...>  earc\n",
      "-0.5800:        <form class=... id=...>  sear\n",
      "-0.5914:              <form action=...>   auth\n",
      "-0.6020:                  form elements  exactly two <input type=text>\n",
      "-0.6555:               <input name=...>   s \n",
      "-0.6672:          <label> TEXT </label>  adresse\n",
      "-0.6714:               <input name=...>   qty \n",
      "-0.7356:  <input type=submit value=...>  reset\n",
      "-0.7451:  <input type=submit value=...>  password\n",
      "-0.7456:  <input type=submit value=...>  send\n",
      "-0.8019:          <label> TEXT </label>  password\n",
      "-0.9132:          <label> TEXT </label>  mail\n",
      "-1.0151:                  form elements  <form method=post\n",
      "-1.0373:               <input name=...>   q \n",
      "-1.2096:                  form elements  no <input type=text>\n",
      "-1.3052:                  form elements  has <select>\n",
      "-1.4406:          <label> TEXT </label>  email\n",
      "-1.6305:                  form elements  exactly two <input type=password>\n",
      "-1.6501:                  form elements  <form method=get\n",
      "-1.8706:                  form elements  exactly one <input type=password>\n",
      "-2.2601:                  form elements  has <textarea>\n",
      "-2.6653:                           bias  bias\n",
      "--------------------------------------------------------------------------------\n",
      "login\n",
      "+3.4824:                  form elements  exactly one <input type=password>\n",
      "+3.0518:              <form action=...>  login\n",
      "+1.8492:  <input type=submit value=...>  submit\n",
      "+1.8024:  <input type=submit value=...>  login\n",
      "+1.4129:  <input type=submit value=...>  in\n",
      "+1.1301:  <input type=submit value=...>  sign in\n",
      "+1.0719:              <form action=...>  openi\n",
      "+1.0719:              <form action=...>  penid\n",
      "+1.0719:              <form action=...>  openid\n",
      "+0.9943:  <input type=submit value=...>  войти\n",
      "+0.8772:  <input type=submit value=...>  sign\n",
      "+0.8643:                  <a> TEXT </a>  password\n",
      "+0.8240:  <input type=submit value=...>  войти войти\n",
      "+0.7895:                  form elements  no <input type=text>\n",
      "+0.7525:          <label> TEXT </label>  remember\n",
      "+0.7180:        <form class=... id=...>   cer\n",
      "+0.7160:              <form action=...>  login#\n",
      "+0.7160:              <form action=...>  ogin# \n",
      "+0.7160:              <form action=...>  gin# \n",
      "+0.7160:              <form action=...>  ogin#\n",
      "+0.7065:              <form action=...>  dlogi\n",
      "+0.7065:              <form action=...>  dlogin\n",
      "+0.7065:              <form action=...>  enidl\n",
      "+0.7065:              <form action=...>  idlogi\n",
      "+0.7065:              <form action=...>  enidlo\n",
      "   ...\n",
      "-0.4937:               <input name=...>   email\n",
      "-0.5260:  <input type=submit value=...>  register now\n",
      "-0.5292:               <input name=...>  email \n",
      "-0.5313:  <input type=submit value=...>  now\n",
      "-0.5474:          <label> TEXT </label>  mail\n",
      "-0.5789:  <input type=submit value=...>  create account\n",
      "-0.5985:                  <a> TEXT </a>  terms service\n",
      "-0.6008:               <input name=...>   q \n",
      "-0.6195:                  <a> TEXT </a>  service\n",
      "-0.6344:                  <a> TEXT </a>  terms\n",
      "-0.7426:                  <a> TEXT </a>  com\n",
      "-0.7458:  <input type=submit value=...>  account\n",
      "-0.8204:  <input type=submit value=...>  create\n",
      "-0.8308:                  form elements  <form method=get\n",
      "-0.9539:                  form elements  has <textarea>\n",
      "-0.9764:  <input type=submit value=...>  regisztráció\n",
      "-0.9843:                  form elements  has <input type=radio>\n",
      "-0.9944:  <input type=submit value=...>  зарегистрироваться\n",
      "-1.0744:  <input type=submit value=...>  register\n",
      "-1.2492:                           bias  bias\n",
      "-1.4652:                  form elements  3 or more <input type=text>\n",
      "-1.6444:                  form elements  exactly two <input type=password>\n",
      "-2.0472:                  form elements  has <input type=email>\n",
      "-2.0521:                  form elements  has <select>\n",
      "-3.0844:                  form elements  no <input type=password>\n",
      "--------------------------------------------------------------------------------\n",
      "order/add to cart\n",
      "+3.2273:               <input name=...>   qty \n",
      "+2.2659:              <form action=...>   cart\n",
      "+1.9485:  <input type=submit value=...>  заказ\n",
      "+1.6613:                  <a> TEXT </a>  email\n",
      "+1.6564:               <input name=...>  list \n",
      "+1.6211:              <form action=...>   prod\n",
      "+1.3928:               <input name=...>  quant\n",
      "+1.3702:               <input name=...>  oncode\n",
      "+1.3702:               <input name=...>  oncod\n",
      "+1.2963:               <input name=...>  ncode\n",
      "+1.2963:               <input name=...>  ncode \n",
      "+1.2336:  <input type=submit value=...>  оплатить\n",
      "+1.2336:  <input type=submit value=...>  оплатить заказ\n",
      "+1.1703:          <label> TEXT </label>  quantity\n",
      "+1.1324:                  <a> TEXT </a>  to\n",
      "+1.1277:                  form elements  has <select>\n",
      "+1.1250:  <input type=submit value=...>  acquista\n",
      "+1.1220:                  <a> TEXT </a>  add to\n",
      "+1.1205:               <input name=...>  uantit\n",
      "+1.1205:               <input name=...>  antit\n",
      "+1.1205:               <input name=...>  antity\n",
      "+1.1205:               <input name=...>  uanti\n",
      "+1.1205:               <input name=...>  quanti\n",
      "+1.1024:               <input name=...>  ntity\n",
      "+1.0852:               <input name=...>  ntity \n",
      "   ...\n",
      "-0.5888:  <input type=submit value=...>  go\n",
      "-0.6032:          <label> TEXT </label>  sort\n",
      "-0.6165:              <form action=...>  event\n",
      "-0.6693:        <form class=... id=...>  form \n",
      "-0.6751:        <form class=... id=...>  orm \n",
      "-0.6799:        <form class=... id=...>  form\n",
      "-0.7144:                  form elements  3 or more <input type=text>\n",
      "-0.7303:  <input type=submit value=...>  search\n",
      "-0.7343:                  form elements  has <input type=checkbox>\n",
      "-0.7370:               <input name=...>  email \n",
      "-0.7600:               <input name=...>   emai\n",
      "-0.7600:               <input name=...>   email\n",
      "-0.7906:               <input name=...>   s \n",
      "-0.7997:               <input name=...>  mail \n",
      "-0.8084:                  form elements  has <textarea>\n",
      "-0.8144:               <input name=...>  email\n",
      "-0.8485:                  form elements  <form method=post\n",
      "-0.9721:                  form elements  has <input type=email>\n",
      "-1.0582:                  form elements  exactly two <input type=password>\n",
      "-1.0602:               <input name=...>   q \n",
      "-1.1429:                  form elements  2 or 3 inputs\n",
      "-1.5612:                  form elements  <form method=get\n",
      "-1.5740:                  form elements  exactly one <input type=password>\n",
      "-1.6992:                  form elements  exactly two <input type=text>\n",
      "-2.4097:                           bias  bias\n",
      "--------------------------------------------------------------------------------\n",
      "other\n",
      "+1.9037:       <input class=... id=...>  from\n",
      "+1.8773:               <input name=...>  surl \n",
      "+1.8286:        <form class=... id=...>   xe \n",
      "+1.5334:               <input name=...>  code \n",
      "+1.5245:                  <a> TEXT </a>  conditions\n",
      "+1.4233:  <input type=submit value=...>  keresés mentése\n",
      "+1.4233:  <input type=submit value=...>  mentése\n",
      "+1.3459:                  form elements  no <input type=text>\n",
      "+1.2918:          <label> TEXT </label>  10\n",
      "+1.2020:                  form elements  no <input type=password>\n",
      "+1.0703:        <form class=... id=...>   add\n",
      "+1.0684:          <label> TEXT </label>  description\n",
      "+1.0561:  <input type=submit value=...>  lähetä\n",
      "+1.0527:              <form action=...>  ers# \n",
      "+1.0502:              <form action=...>  quest\n",
      "+1.0229:               <input name=...>   url \n",
      "+1.0021:                  form elements  has <input type=radio>\n",
      "+0.9992:        <form class=... id=...>  jump\n",
      "+0.9757:              <form action=...>  tiona\n",
      "+0.9741:  <input type=submit value=...>  continue\n",
      "+0.9583:          <label> TEXT </label>  zip\n",
      "+0.9566:              <form action=...>  event\n",
      "+0.9392:          <label> TEXT </label>  line\n",
      "+0.9233:        <form class=... id=...>  lect \n",
      "+0.9216:  <input type=submit value=...>  keresés\n",
      "   ...\n",
      "-0.9848:                  <a> TEXT </a>  forums\n",
      "-0.9851:          <label> TEXT </label>  search\n",
      "-0.9863:                  <a> TEXT </a>  statement\n",
      "-0.9926:                  form elements  has <select>\n",
      "-0.9995:          <label> TEXT </label>  year\n",
      "-1.0110:                  <a> TEXT </a>  affiliate\n",
      "-1.0181:               <input name=...>  ubmit \n",
      "-1.0181:               <input name=...>  bmit \n",
      "-1.0359:               <input name=...>  submit\n",
      "-1.0359:               <input name=...>  ubmit\n",
      "-1.0603:               <input name=...>  email\n",
      "-1.0634:               <input name=...>  submi\n",
      "-1.0762:               <input name=...>  mail \n",
      "-1.1114:               <input name=...>   qty \n",
      "-1.1735:                           bias  bias\n",
      "-1.2461:               <input name=...>  list \n",
      "-1.3485:                  form elements  has <input type=email>\n",
      "-1.4211:                  form elements  exactly one <input type=password>\n",
      "-1.4414:  <input type=submit value=...>  search\n",
      "-1.5393:              <form action=...>  login\n",
      "-1.6248:               <input name=...>   s \n",
      "-1.7547:          <label> TEXT </label>  sort\n",
      "-1.7813:                  form elements  exactly one <input type=text>\n",
      "-1.8461:                  form elements  has <textarea>\n",
      "-2.1208:               <input name=...>   q \n",
      "--------------------------------------------------------------------------------\n",
      "password/login recovery\n",
      "+2.2791:  <input type=submit value=...>  reset\n",
      "+2.2555:  <input type=submit value=...>  password\n",
      "+1.6552:  <input type=submit value=...>  priminti\n",
      "+1.5349:  <input type=submit value=...>  reset password\n",
      "+1.4216:               <input name=...>  mail \n",
      "+1.4083:          <label> TEXT </label>  email\n",
      "+1.3379:        <form class=... id=...>  pass\n",
      "+1.3282:  <input type=submit value=...>  odošli\n",
      "+1.2716:               <input name=...>  email\n",
      "+1.2530:              <form action=...>  passw\n",
      "+1.2530:              <form action=...>  passwo\n",
      "+1.2530:              <form action=...>  sswor\n",
      "+1.2530:              <form action=...>  asswor\n",
      "+1.2530:              <form action=...>  asswo\n",
      "+1.1767:              <form action=...>  sword\n",
      "+1.1767:              <form action=...>  ssword\n",
      "+1.1495:              <form action=...>  forgot\n",
      "+1.1495:              <form action=...>  forgo\n",
      "+1.1495:              <form action=...>  orgot\n",
      "+1.1485:               <input name=...>  email \n",
      "+1.1360:               <input name=...>  login \n",
      "+1.1360:               <input name=...>  ogin \n",
      "+1.0991:               <input name=...>   logi\n",
      "+1.0991:               <input name=...>   login\n",
      "+1.0666:  <input type=submit value=...>  次へ\n",
      "   ...\n",
      "-0.5267:  <input type=submit value=...>  search\n",
      "-0.5274:        <form class=... id=...>  _form\n",
      "-0.5324:                  form elements  exactly two <input type=password>\n",
      "-0.5474:              <form action=...>   sign\n",
      "-0.5553:              <form action=...>   signu\n",
      "-0.5687:  <input type=submit value=...>  account\n",
      "-0.6057:                  form elements  <form method=post\n",
      "-0.6595:  <input type=submit value=...>  sign\n",
      "-0.6912:               <input name=...>   qty \n",
      "-0.7177:  <input type=submit value=...>  register\n",
      "-0.7281:              <form action=...>  signup\n",
      "-0.7281:              <form action=...>  signu\n",
      "-0.7281:              <form action=...>  ignup\n",
      "-0.7864:                  form elements  3 or more <input type=text>\n",
      "-0.8368:  <input type=submit value=...>  подписаться\n",
      "-0.8690:  <input type=submit value=...>  subscribe\n",
      "-0.9040:               <input name=...>   q \n",
      "-0.9910:                  form elements  no <input type=text>\n",
      "-1.3537:                  form elements  has <input type=checkbox>\n",
      "-1.3566:                  form elements  has <input type=radio>\n",
      "-1.6953:                  form elements  has <select>\n",
      "-2.1420:                  form elements  <form method=get\n",
      "-2.2118:                  form elements  has <textarea>\n",
      "-2.7477:                           bias  bias\n",
      "-2.8691:                  form elements  exactly one <input type=password>\n",
      "--------------------------------------------------------------------------------\n",
      "registration\n",
      "+3.1553:                  form elements  exactly two <input type=password>\n",
      "+2.5143:  <input type=submit value=...>  register\n",
      "+1.7614:  <input type=submit value=...>  create\n",
      "+1.5616:  <input type=submit value=...>  registrovať\n",
      "+1.4739:              <form action=...>  regis\n",
      "+1.4635:              <form action=...>  regist\n",
      "+1.4635:              <form action=...>  egist\n",
      "+1.4355:              <form action=...>  giste\n",
      "+1.4355:              <form action=...>  gister\n",
      "+1.4355:              <form action=...>  ister\n",
      "+1.4355:              <form action=...>  egiste\n",
      "+1.3142:  <input type=submit value=...>  regisztráció\n",
      "+1.2229:                  form elements  has <input type=email>\n",
      "+1.1371:  <input type=submit value=...>  account\n",
      "+1.0963:  <input type=submit value=...>  зарегистрироваться\n",
      "+1.0892:                  form elements  3 or more <input type=text>\n",
      "+1.0521:              <form action=...>  ster#\n",
      "+1.0521:              <form action=...>  ster# \n",
      "+1.0521:              <form action=...>  ister#\n",
      "+1.0040:              <form action=...>  ter# \n",
      "+0.9866:              <form action=...>  signup\n",
      "+0.9866:              <form action=...>  signu\n",
      "+0.9866:              <form action=...>  ignup\n",
      "+0.9810:               <input name=...>  emailc\n",
      "+0.9810:               <input name=...>  mailc\n",
      "   ...\n",
      "-0.5715:              <form action=...>  ogin# \n",
      "-0.5715:              <form action=...>  ogin#\n",
      "-0.5717:  <input type=submit value=...>  search\n",
      "-0.5760:  <input type=submit value=...>  submit\n",
      "-0.5885:        <form class=... id=...>  form\n",
      "-0.6094:  <input type=submit value=...>  reset password\n",
      "-0.6151:                  <a> TEXT </a>  forgot\n",
      "-0.6482:          <label> TEXT </label>  email\n",
      "-0.6754:  <input type=submit value=...>  reset\n",
      "-0.6775:                  <a> TEXT </a>  conditions\n",
      "-0.6818:                  form elements  has <input type=checkbox>\n",
      "-0.7101:                  form elements  2 or 3 inputs\n",
      "-0.7876:  <input type=submit value=...>  in\n",
      "-0.8990:                  <a> TEXT </a>  password\n",
      "-0.9133:              <form action=...>  login\n",
      "-0.9426:                  <a> TEXT </a>  пароль\n",
      "-0.9440:                           bias  bias\n",
      "-0.9732:  <input type=submit value=...>  password\n",
      "-1.0133:                  form elements  <form method=get\n",
      "-1.1063:                  form elements  exactly one <input type=text>\n",
      "-1.1649:                  form elements  exactly one <input type=password>\n",
      "-1.2750:  <input type=submit value=...>  login\n",
      "-1.5632:                  form elements  has <textarea>\n",
      "-1.6221:                  form elements  no <input type=text>\n",
      "-2.9856:                  form elements  no <input type=password>\n",
      "--------------------------------------------------------------------------------\n",
      "search\n",
      "+3.0135:               <input name=...>   q \n",
      "+2.5323:               <input name=...>   s \n",
      "+2.0315:          <label> TEXT </label>  sort\n",
      "+1.8639:  <input type=submit value=...>  search\n",
      "+1.7978:              <form action=...>  &sid=\n",
      "+1.6596:                  form elements  has <select>\n",
      "+1.5274:               <input name=...>  earch\n",
      "+1.5274:               <input name=...>  search\n",
      "+1.5274:               <input name=...>  searc\n",
      "+1.4988:          <label> TEXT </label>  year\n",
      "+1.3743:              <form action=...>  earch\n",
      "+1.3743:              <form action=...>  search\n",
      "+1.3743:              <form action=...>  searc\n",
      "+1.3473:       <input class=... id=...>  arch\n",
      "+1.3071:        <form class=... id=...>  efor\n",
      "+1.3071:        <form class=... id=...>  eform\n",
      "+1.2918:               <input name=...>  text \n",
      "+1.2533:       <input class=... id=...>   q \n",
      "+1.2510:        <form class=... id=...>  searc\n",
      "+1.2510:        <form class=... id=...>  sear\n",
      "+1.2510:        <form class=... id=...>  arch\n",
      "+1.2510:        <form class=... id=...>  earc\n",
      "+1.2510:        <form class=... id=...>  earch\n",
      "+1.2480:               <input name=...>   searc\n",
      "+1.2480:               <input name=...>   sear\n",
      "   ...\n",
      "-0.7735:          <label> TEXT </label>  email\n",
      "-0.8059:  <input type=submit value=...>  sign\n",
      "-0.8110:        <form class=... id=...>  ect \n",
      "-0.8171:                  form elements  has <input type=radio>\n",
      "-0.8190:        <form class=... id=...>  lect \n",
      "-0.9088:              <form action=...>  event\n",
      "-0.9395:                  form elements  3 or more <input type=text>\n",
      "-1.0058:                  form elements  exactly two <input type=password>\n",
      "-1.0126:              <form action=...>   cart\n",
      "-1.0183:                  form elements  has <input type=email>\n",
      "-1.0371:               <input name=...>  surl \n",
      "-1.1587:  <input type=submit value=...>  submit\n",
      "-1.2916:               <input name=...>  email \n",
      "-1.3056:        <form class=... id=...>   xe \n",
      "-1.3097:       <input class=... id=...>  from\n",
      "-1.3891:                  form elements  no <input type=text>\n",
      "-1.3934:               <input name=...>  mail \n",
      "-1.5475:               <input name=...>   emai\n",
      "-1.5475:               <input name=...>   email\n",
      "-1.5511:               <input name=...>  email\n",
      "-1.6066:               <input name=...>   qty \n",
      "-1.8007:                  form elements  <form method=post\n",
      "-1.8374:                           bias  bias\n",
      "-1.9482:                  form elements  exactly one <input type=password>\n",
      "-2.1162:                  form elements  has <textarea>\n",
      "--------------------------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "model.fit(X, y)\n",
    "print_informative_features(FEATURES, clf, 25)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Compare results with \"loginform\" library\n",
    "\n",
    "It is not possible to compare the results with \"loginform\" library directly because loginform\n",
    "\n",
    "* always tries to return a login form even if the score is low;\n",
    "* only detects login forms;\n",
    "* in case of several forms returns a single form with the best score instead of deciding for each form whether to return it or not.\n",
    "\n",
    "So we used two approaches:\n",
    "\n",
    "1. Use `loginform._form_score` with different thresholds; assume that if score is greater than or equal to a threshold `loginform` detected a login form.\n",
    "2. Train the same model, but using features from loginform library (weights will be learned instead of being hardcoded as 'score' increments/decrements).\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. loginform scores + thresholds\n",
    "\n",
    "* **score >= -100** means \"simply treat all forms as login forms\".\n",
    "\n",
    "* **score >= 0** all (or most) login forms are captured, but there are many false positives. \n",
    "  It is only slightly better than treating all forms as login forms.\n",
    "\n",
    "* **score >= 10** F1 score is the best among all thresholds, \n",
    "  but the quality is significantly worse than F1 of ML-based models.\n",
    "  \n",
    "* **score >= 20** ~90% of detected login forms are correct, but most \n",
    "  login forms are not detected. Also, ~90% number is still lower than what ML-based models give us.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "score >= -100:    precision = 0.178    recall = 1.000    F1 = 0.303\n",
      "score >=  -10:    precision = 0.201    recall = 0.973    F1 = 0.334\n",
      "score >=    0:    precision = 0.322    recall = 0.968    F1 = 0.484\n",
      "score >=   10:    precision = 0.710    recall = 0.869    F1 = 0.781\n",
      "score >=   20:    precision = 0.855    recall = 0.320    F1 = 0.466\n",
      "score >=   30:    precision = 1.000    recall = 0.009    F1 = 0.018\n",
      "CPU times: user 2.35 s, sys: 6.9 ms, total: 2.36 s\n",
      "Wall time: 2.45 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/kmike/envs/dl/lib/python3.5/site-packages/sklearn/metrics/classification.py:1074: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "import loginform\n",
    "\n",
    "def labels_to_binary(y):\n",
    "    \"\"\" Convert labels to 2-classes: login forms and non-login forms \"\"\"\n",
    "    return [tp == 'login' for tp in y]\n",
    "\n",
    "    \n",
    "def predict_loginform(X, threshold):\n",
    "    \"\"\"\n",
    "    Return if forms are login or not using loginform\n",
    "    library scores and a threshold.\n",
    "    \"\"\"\n",
    "    return [\n",
    "        (loginform._form_score(form) >= threshold)\n",
    "        for form in X\n",
    "    ]\n",
    "\n",
    "\n",
    "def print_threshold_metrics(X, y, threshold):\n",
    "    y = labels_to_binary(y)\n",
    "    y_pred = predict_loginform(X, threshold)\n",
    "\n",
    "    precision, recall, f1, support = precision_recall_fscore_support(y, y_pred, pos_label=True)\n",
    "    print(\n",
    "        \"score >= %4d:    precision = %0.3f    recall = %0.3f    F1 = %0.3f\" % (\n",
    "        threshold, precision[1], recall[1], f1[1]\n",
    "    ))\n",
    "\n",
    "\n",
    "for threshold in [-100, -10, 0, 10, 20, 30]:\n",
    "    print_threshold_metrics(X, y, threshold)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Use loginform features, but autodetect scores\n",
    "\n",
    "The following ML-based model is trained using original loginform features (conditions used to increase or decrease the score). Roughly speaking, it uses the same information as loginform library, but instead of hardcoding `score += 10` and `score -= 10` the numbers are adjusted based on training data.\n",
    "\n",
    "Note that the login form detection quality is significantly better than the quality of threshold-based model; it is only slightly worse than the quality of a \"full\" forms detection model. This means original loginform features are quite good at detecting login forms. But for other form types these features are not enough: other scores are bad."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                   precision    recall  f1-score   support\n",
      "\n",
      "           search       0.55      0.79      0.65       364\n",
      "            login       0.92      0.95      0.93       222\n",
      "     registration       0.95      0.67      0.78       153\n",
      "  contact/comment       0.47      0.73      0.58       120\n",
      "join mailing list       0.11      0.01      0.02       107\n",
      "            other       0.38      0.59      0.46       122\n",
      "\n",
      "      avg / total       0.61      0.70      0.64      1088\n",
      "\n",
      "61.3% forms are classified correctly.\n",
      "contact/comment\n",
      "+3.6044:                      loginform  typecount_password_0\n",
      "+2.9594:                      loginform  typecount_text_gt1\n",
      "+0.1378:                      loginform  typecount_text_0\n",
      "+0.0474:                      loginform  typecount_radio_gt0\n",
      "-1.3172:                      loginform  typecount_checkbox_gt1\n",
      "-1.4733:                      loginform  typecount_password_eq1\n",
      "-1.5214:                      loginform  2_or_3_inputs\n",
      "-6.5001:                           bias  bias\n",
      "--------------------------------------------------------------------------------\n",
      "join mailing list\n",
      "+2.4583:                      loginform  typecount_password_0\n",
      "+1.1901:                      loginform  typecount_checkbox_gt1\n",
      "+0.3436:                      loginform  2_or_3_inputs\n",
      "-0.2444:                      loginform  typecount_text_0\n",
      "-0.7037:                      loginform  typecount_text_gt1\n",
      "-0.7194:                      loginform  typecount_radio_gt0\n",
      "-1.9839:                      loginform  typecount_password_eq1\n",
      "-4.4473:                           bias  bias\n",
      "--------------------------------------------------------------------------------\n",
      "login\n",
      "+5.6971:                      loginform  typecount_password_eq1\n",
      "+0.9164:                      loginform  typecount_text_0\n",
      "+0.3639:                      loginform  2_or_3_inputs\n",
      "-1.8354:                      loginform  typecount_password_0\n",
      "-1.9723:                      loginform  typecount_checkbox_gt1\n",
      "-2.0403:                      loginform  typecount_radio_gt0\n",
      "-2.2831:                      loginform  typecount_text_gt1\n",
      "-2.7096:                           bias  bias\n",
      "--------------------------------------------------------------------------------\n",
      "order/add to cart\n",
      "+1.7240:                      loginform  typecount_password_0\n",
      "+0.4020:                      loginform  typecount_radio_gt0\n",
      "+0.0433:                      loginform  typecount_text_0\n",
      "-0.5080:                      loginform  2_or_3_inputs\n",
      "-0.8300:                      loginform  typecount_checkbox_gt1\n",
      "-1.0047:                      loginform  typecount_text_gt1\n",
      "-2.0615:                      loginform  typecount_password_eq1\n",
      "-4.0580:                           bias  bias\n",
      "--------------------------------------------------------------------------------\n",
      "other\n",
      "+2.5668:                      loginform  typecount_text_0\n",
      "+2.0098:                      loginform  typecount_password_0\n",
      "+0.9708:                      loginform  typecount_text_gt1\n",
      "+0.8658:                      loginform  typecount_radio_gt0\n",
      "+0.0172:                      loginform  typecount_checkbox_gt1\n",
      "-0.1391:                      loginform  2_or_3_inputs\n",
      "-2.0877:                      loginform  typecount_password_eq1\n",
      "-5.0781:                           bias  bias\n",
      "--------------------------------------------------------------------------------\n",
      "password/login recovery\n",
      "+1.2268:                      loginform  typecount_password_0\n",
      "+0.7331:                      loginform  2_or_3_inputs\n",
      "-0.3935:                      loginform  typecount_text_gt1\n",
      "-0.6895:                      loginform  typecount_text_0\n",
      "-2.0990:                      loginform  typecount_checkbox_gt1\n",
      "-2.3290:                      loginform  typecount_radio_gt0\n",
      "-2.7624:                      loginform  typecount_password_eq1\n",
      "-3.3845:                           bias  bias\n",
      "--------------------------------------------------------------------------------\n",
      "registration\n",
      "+2.1473:                      loginform  typecount_text_gt1\n",
      "+1.5049:                           bias  bias\n",
      "+0.6985:                      loginform  typecount_radio_gt0\n",
      "+0.6783:                      loginform  typecount_checkbox_gt1\n",
      "+0.5736:                      loginform  typecount_text_0\n",
      "+0.0624:                      loginform  2_or_3_inputs\n",
      "-4.3932:                      loginform  typecount_password_eq1\n",
      "-5.9793:                      loginform  typecount_password_0\n",
      "--------------------------------------------------------------------------------\n",
      "search\n",
      "+3.7243:                      loginform  typecount_password_0\n",
      "+0.6919:                      loginform  typecount_checkbox_gt1\n",
      "-0.0865:                      loginform  2_or_3_inputs\n",
      "-0.3891:                      loginform  typecount_radio_gt0\n",
      "-1.2802:                      loginform  typecount_text_0\n",
      "-2.1901:                      loginform  typecount_text_gt1\n",
      "-2.4596:                      loginform  typecount_password_eq1\n",
      "-3.4671:                           bias  bias\n",
      "--------------------------------------------------------------------------------\n",
      "CPU times: user 4.48 s, sys: 5.43 ms, total: 4.49 s\n",
      "Wall time: 4.5 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "LOGINFORM_FEATURES = [\n",
    "    ('bias', features.Bias(), DictVectorizer()),\n",
    "    ('loginform', features.OldLoginformFeatures(), DictVectorizer())\n",
    "]\n",
    "# loginform_clf = LinearSVC(C=0.5, fit_intercept=False)\n",
    "loginform_clf = LogisticRegression(penalty='l2', C=5, fit_intercept=False, random_state=0)\n",
    "\n",
    "model = make_pipeline(\n",
    "    _create_feature_union(LOGINFORM_FEATURES), \n",
    "    loginform_clf,\n",
    ")\n",
    "\n",
    "formtype_model.print_classification_report(annotations, n_folds=10, model=model)\n",
    "\n",
    "model.fit(X, y)\n",
    "print_informative_features(LOGINFORM_FEATURES, loginform_clf, 25)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
