{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.insert(0, '..')\n",
    "\n",
    "from formasaurus import features, evaluation, tool\n",
    "from formasaurus.storage import Storage, FORM_TYPES, FORM_TYPES_INV, load_html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Annotated HTML forms:\n",
      "\n",
      "177   search                    (s)\n",
      "139   login                     (l)\n",
      "111   other                     (o)\n",
      "89    registration              (r)\n",
      "50    password/login recovery   (p)\n",
      "38    join mailing list         (m)\n",
      "36    contact                   (c)\n",
      "\n",
      "Total form count: 640\n"
     ]
    }
   ],
   "source": [
    "storage = Storage(\"../formasaurus/data\")\n",
    "storage.print_type_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": []
    }
   ],
   "source": [
    "TRAIN_SIZE = 400\n",
    "\n",
    "X, y = storage.get_Xy(verbose=True)\n",
    "X_train, X_test, y_train, y_test = X[:TRAIN_SIZE], X[TRAIN_SIZE:], y[:TRAIN_SIZE], y[TRAIN_SIZE:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "mailing_list_forms = [form for form, tp in zip(X_test, y_test) if tp == 'm']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<form id=\"newsletterForm\" action=\"/profile/subscribe_home/\" method=\"post\">            Hírlevél feliratkozás\n",
      "<input tabindex=\"100\" name=\"RegistrationForm[username]\" id=\"RegistrationForm_username\" type=\"text\">\n",
      "<input tabindex=\"101\" name=\"RegistrationForm[email]\" id=\"RegistrationForm_email\" type=\"text\">\n",
      "<input tabindex=\"101\" id=\"submit_subscribe\" class=\"submit_subscribe\" type=\"submit\" name=\"yt0\" value=\"\">\n",
      "</form>\n",
      "========================================\n",
      "<form id=\"newsletterForm\" action=\"/profile/subscribe_home/\" method=\"post\">            Hírlevél feliratkozás\n",
      "<input tabindex=\"100\" name=\"RegistrationForm[username]\" id=\"RegistrationForm_username\" type=\"text\">\n",
      "<input tabindex=\"101\" name=\"RegistrationForm[email]\" id=\"RegistrationForm_email\" type=\"text\">\n",
      "<input tabindex=\"101\" id=\"submit_subscribe\" class=\"submit_subscribe\" type=\"submit\" name=\"yt1\" value=\"\">\n",
      "</form>\n",
      "========================================\n",
      "<form action=\"http://petswelcome.list-manage.com/subscribe/post?u=ade3865e57924e2f40e3e555a&amp;id=273168a6fe\" method=\"post\" id=\"mc-embedded-subscribe-form\" name=\"mc-embedded-subscribe-form\" class=\"validate\" target=\"_blank\">\n",
      "<input type=\"text\" value=\"Email Address\" name=\"EMAIL\" class=\"required email\" id=\"mce-EMAIL\">\n",
      "<input type=\"text\" value=\"First Name\" name=\"FNAME\" class=\"required\" id=\"mce-FNAME\">\n",
      "<input type=\"submit\" value=\"Subscribe\" name=\"subscribe\" id=\"mc-embedded-subscribe\" class=\"btn\">\n",
      "</form>\n",
      "========================================\n",
      "<form class=\"newsletter\" action=\"\">\n",
      "<input type=\"hidden\" name=\"user_id\" value=\"\">\n",
      "<input type=\"hidden\" name=\"user_email\" value=\"\">\n",
      "<input type=\"text\" name=\"subscriber\" value=\"\" id=\"subscriber_email\">\n",
      "<input type=\"submit\" name=\"submit\" value=\"Noriu gauti naujienlaiškius\">\n",
      "</form>\n",
      "========================================\n",
      "<form id=\"subscription-form\" method=\"POST\" class=\"form\" action=\"/subscription/add\">\n",
      "<input type=\"text\" name=\"email\" class=\"required email\">\n",
      "<select id=\"city_id\" name=\"city_id\"><option value=\"\">-- Select City --</option>\n",
      "<option value=\"9\">Albany</option>\n",
      "<option value=\"58\">Albuquerque</option>\n",
      "<option value=\"110\">Alexandria Louisiana</option>\n",
      "<option value=\"59\">Amarillo</option>\n",
      "<option value=\"122\">Athens</option>\n",
      "<option value=\"3\">Atlanta</option>\n",
      "<option value=\"125\">Atlantic City</option>\n",
      "<option value=\"7\">Austin</option>\n",
      "<option value=\"23\">Baltimore</option>\n",
      "<option value=\"46\">Baton Rouge</option>\n",
      "<option value=\"111\">Beaumont</option>\n",
      "<option value=\"53\">Big Island</option>\n",
      "<option value=\"64\">Boise</option>\n",
      "<option value=\"6\">Boston</option>\n",
      "<option value=\"55\">Buffalo</option>\n",
      "<option value=\"65\">Cedar Rapids</option>\n",
      "<option value=\"118\">Cedar Valley</option>\n",
      "<option value=\"66\">Charleston</option>\n",
      "<option value=\"8\">Charlotte</option>\n",
      "<option value=\"67\">Chattanooga</option>\n",
      "<option value=\"34\">Chicago</option>\n",
      "<option value=\"68\">Cincinnati</option>\n",
      "<option value=\"32\">Cleveland</option>\n",
      "<option value=\"69\">Colorado Springs</option>\n",
      "<option value=\"30\">Columbus</option>\n",
      "<option value=\"20\">Dallas</option>\n",
      "<option value=\"70\">Dayton</option>\n",
      "<option value=\"4\">Denver</option>\n",
      "<option value=\"27\">Detroit</option>\n",
      "<option value=\"33\">East Bay</option>\n",
      "<option value=\"71\">El Paso</option>\n",
      "<option value=\"72\">Eugene</option>\n",
      "<option value=\"42\">Fort Worth</option>\n",
      "<option value=\"252\">Fort-lauderdale</option>\n",
      "<option value=\"73\">Fresno</option>\n",
      "<option value=\"74\">Ft. Myers</option>\n",
      "<option value=\"75\">Ft. Wayne</option>\n",
      "<option value=\"108\">Golden Triangle</option>\n",
      "<option value=\"76\">Grand Rapids</option>\n",
      "<option value=\"77\">Green Bay</option>\n",
      "<option value=\"78\">Greenwood</option>\n",
      "<option value=\"109\">Gulfport</option>\n",
      "<option value=\"79\">Harrisburg</option>\n",
      "<option value=\"80\">Hartford</option>\n",
      "<option value=\"52\">Honolulu</option>\n",
      "<option value=\"17\">Houston</option>\n",
      "<option value=\"22\">Indianapolis</option>\n",
      "<option value=\"12\">Jacksonville</option>\n",
      "<option value=\"48\">Johnson County</option>\n",
      "<option value=\"14\">Kansas City</option>\n",
      "<option value=\"81\">Knoxville</option>\n",
      "<option value=\"106\">Lafayette</option>\n",
      "<option value=\"105\">Lake Charles</option>\n",
      "<option value=\"11\">Las Vegas</option>\n",
      "<option value=\"117\">Lawrence</option>\n",
      "<option value=\"82\">Little Rock</option>\n",
      "<option value=\"49\">Los Angeles/ventura County</option>\n",
      "<option value=\"28\">Louisville</option>\n",
      "<option value=\"83\">Lubbock</option>\n",
      "<option value=\"116\">Manhattan - Kansas</option>\n",
      "<option value=\"54\">Maui</option>\n",
      "<option value=\"47\">Memphis</option>\n",
      "<option value=\"13\">Miami</option>\n",
      "<option value=\"84\">Milwaukee</option>\n",
      "<option value=\"85\">Minneapolis / St Paul</option>\n",
      "<option value=\"86\">Mobile</option>\n",
      "<option value=\"87\">Montgomery</option>\n",
      "<option value=\"25\">Nashville</option>\n",
      "<option value=\"36\" selected>National Deals</option>\n",
      "<option value=\"43\">New Orleans</option>\n",
      "<option value=\"5\">New York City</option>\n",
      "<option value=\"88\">Norfolk</option>\n",
      "<option value=\"57\">Northwest Suburbs - Chicago</option>\n",
      "<option value=\"89\">Odessa</option>\n",
      "<option value=\"44\">Oklahoma City</option>\n",
      "<option value=\"90\">Omaha</option>\n",
      "<option value=\"29\">Orange County</option>\n",
      "<option value=\"2\">Orlando</option>\n",
      "<option value=\"91\">Palm Springs</option>\n",
      "<option value=\"121\">Pets</option>\n",
      "<option value=\"31\">Philadelphia</option>\n",
      "<option value=\"18\">Phoenix</option>\n",
      "<option value=\"92\">Pittsburgh</option>\n",
      "<option value=\"16\">Portland</option>\n",
      "<option value=\"93\">Portland Maine</option>\n",
      "<option value=\"38\">Poughkeepsie</option>\n",
      "<option value=\"94\">Providence</option>\n",
      "<option value=\"61\">Raleigh</option>\n",
      "<option value=\"95\">Richmond</option>\n",
      "<option value=\"56\">Rochester</option>\n",
      "<option value=\"113\">Rochester - Minnesota</option>\n",
      "<option value=\"96\">Sacramento</option>\n",
      "<option value=\"45\">Salt Lake City</option>\n",
      "<option value=\"26\">San Antonio</option>\n",
      "<option value=\"19\">San Diego</option>\n",
      "<option value=\"50\">San Fernando Valley</option>\n",
      "<option value=\"10\">San Francisco</option>\n",
      "<option value=\"21\">San Jose</option>\n",
      "<option value=\"51\">Santa Clarita</option>\n",
      "<option value=\"24\">Seattle</option>\n",
      "<option value=\"107\">Shreveport</option>\n",
      "<option value=\"115\">Sioux City</option>\n",
      "<option value=\"97\">Sioux Falls</option>\n",
      "<option value=\"98\">Springfield</option>\n",
      "<option value=\"60\">St. Louis</option>\n",
      "<option value=\"37\">St. Petersburg</option>\n",
      "<option value=\"112\">Sugar Land</option>\n",
      "<option value=\"99\">Syracuse</option>\n",
      "<option value=\"100\">Tacoma</option>\n",
      "<option value=\"101\">Tallahassee</option>\n",
      "<option value=\"1\">Tampa Bay</option>\n",
      "<option value=\"123\">Temecula</option>\n",
      "<option value=\"114\">Topeka</option>\n",
      "<option value=\"41\">Tucson</option>\n",
      "<option value=\"40\">Tulsa</option>\n",
      "<option value=\"39\">Utica</option>\n",
      "<option value=\"104\">Washington D. C.</option></select>\n",
      "Submit\n",
      "<input type=\"hidden\" name=\"url\" value=\"/national/share\">\n",
      "</form>\n",
      "========================================\n",
      "<form action=\"https://www.rakuten.de/kundenkonto/newsletter/anmelden\" method=\"post\">\n",
      "<input type=\"text\" name=\"newsletter\" id=\"newsletter\" class=\"tr_footer_newsletter_input\" value=\"Ihre E-Mail-Adresse\" target=\"https://www.rakuten.de/kundenkonto/newsletter/anmelden\">\n",
      "<input type=\"submit\" id=\"button\" value=\"OK\" class=\"tr_footer_newsletter_icon\">\n",
      "</form>\n",
      "========================================\n",
      "<form action=\"http://www.rakuten.de/kundenkonto/newsletter/anmelden\" method=\"post\" class=\"newsletter-register\">\n",
      "<input type=\"text\" id=\"nl_email\" name=\"susa_newsletter\">\n",
      "Jetzt anmelden &amp; sparen\n",
      "</form>\n",
      "========================================\n",
      "<form action=\"http://www.rakuten.de/kundenkonto/newsletter/anmelden\" method=\"post\" class=\"newsletter-register\">\n",
      "<input type=\"text\" id=\"nl_email\" name=\"susa_newsletter\" value=\"Ihre Email-Adresse eingeben\">\n",
      "Anmelden\n",
      "<input type=\"hidden\" name=\"inp_40840\" value=\"DE_2014\">\n",
      "</form>\n",
      "========================================\n",
      "<form id=\"form_news\" name=\"form_news\"> Recevoir la newsletter bimensuelle de routard.com  <input id=\"email_news\" name=\"email_news\" type=\"text\" value=\"Votre adresse email\">              <a href=\"/pages_spe/newsletter\" target=\"_blank\">consulter un exemple</a>    </form>\n",
      "========================================\n",
      "<form name=\"ccoptin\" action=\"http://visitor.r20.constantcontact.com/d.jsp\" target=\"_blank\" method=\"post\"> Sign up for our Happy Homes Email Newsletter  <input type=\"text\" name=\"ea\" size=\"16\" value=\"\"> <input type=\"submit\" name=\"go\" value=\"GO\"> <input type=\"hidden\" name=\"llr\" value=\"uxruvzcab\"> <input type=\"hidden\" name=\"m\" value=\"1102520665844\"> <input type=\"hidden\" name=\"p\" value=\"oi\"> </form>\n",
      "========================================\n",
      "<form action=\"http://trendo.bg/user/signup/flybox\" method=\"POST\" id=\"signup_flybox\">\n",
      "<input type=\"text\" name=\"signup[email]\" value=\"\" class=\"form-control\">\n",
      "<input type=\"hidden\" name=\"signup[gender]\" value=\"\">\n",
      "<input type=\"submit\" value=\" Мъж\" class=\"btn btn-default\">\n",
      "<input type=\"submit\" value=\"Жена\" class=\"btn btn-default\">\n",
      "</form>\n",
      "========================================\n",
      "<form method=\"get\" action=\"/web/en/emailSubscription.page\">\n",
      "<input type=\"text\" id=\"signupText\" class=\"mr20 empty\" name=\"emailsignup\" value=\"Enter e-mail address\">\n",
      "<input type=\"submit\" id=\"signupButton\" name=\"btnFooterSignUp\" value=\"Sign up\">\n",
      "<input type=\"hidden\" value=\"e\" name=\"cmtag\">\n",
      "</form>\n",
      "========================================\n",
      "<form action=\"https://wpmudev.us1.list-manage.com/subscribe/post?u=4e8bfa5d84735d9c9d7016040&amp;id=1ba864f654\" method=\"post\" id=\"mc-embedded-subscribe-form\" name=\"mc-embedded-subscribe-form\" class=\"validate\" target=\"_blank\">\n",
      "<label for=\"newsletter-email\">Sign up for the WPMU DEV Newsletter!</label>\n",
      "<input value=\"\" name=\"EMAIL\" class=\"required email\" id=\"mce-EMAIL\" type=\"text\">\n",
      "<input type=\"submit\" class=\"submit\" value=\"Submit\" id=\"wpmu-mailchimp-send\">\n",
      "</form>\n",
      "========================================\n",
      "<form id=\"newsletterValidateDetail\" method=\"post\" action=\"/newsletter/subscriber/new\" name=\"footer.newsletter\">\n",
      "<input type=\"hidden\" value=\"1\" name=\"wt_form\"> <input type=\"text\" class=\"email-correction inputText required-entry validate-email prefilled\" value=\"E-Mail-Adresse hier eingeben\" name=\"email\" id=\"newsletter\">\n",
      "Für Frauen\n",
      "Für Männer\n",
      "<input type=\"hidden\" name=\"_xtk\" value=\"m50vxeTVWWp9r5LNVDcYpZwcTFpFrEX6pvw9IUepK6EY1uFT3l3opqqQfsfuvYpb\">\n",
      "</form>\n",
      "========================================\n",
      "<form id=\"newsletterValidateDetail\" method=\"post\" action=\"/newsletter/subscriber/new\" name=\"footer.newsletter\">\n",
      "<input type=\"hidden\" value=\"1\" name=\"wt_form\"> <input type=\"text\" class=\"email-correction inputText required-entry validate-email prefilled\" value=\"E-Mail-Adresse hier eingeben\" name=\"email\" id=\"newsletter\">\n",
      "Für Frauen\n",
      "Für Männer\n",
      "<input type=\"hidden\" name=\"_xtk\" value=\"gM9xzfZWpPVdsDP1yb2u5FRcSxxA3MOjR3suJmfK8FhluuHjS0QMJIRq5B0kzBPN\">\n",
      "</form>\n",
      "========================================\n",
      "<form id=\"newsletterValidateDetail\" method=\"post\" action=\"/newsletter/subscriber/new\" name=\"footer.newsletter\">\n",
      "<input type=\"hidden\" value=\"1\" name=\"wt_form\"> <input type=\"text\" class=\"email-correction inputText required-entry validate-email prefilled\" value=\"E-Mail-Adresse hier eingeben\" name=\"email\" id=\"newsletter\">\n",
      "Für Frauen\n",
      "Für Männer\n",
      "<input type=\"hidden\" name=\"_xtk\" value=\"KydAFS3DVnsPWyVjSb51dftpgpvbtfQOgj708t8fcGjXnsyhDtdcQYCjr7b9sN78\">\n",
      "</form>\n",
      "========================================\n",
      "<form id=\"page_footer_iaFooter_emailSignup-form\" class=\"AcAJ-form\" method=\"post\" action=\"http://www.zazzle.com/eml/signupdialog\" name=\"emailSignup\" target=\"emailSignupWindow\" accept-charset=\"UTF-8\">\n",
      "<input id=\"page_footer_iaFooter_emailSignup_elements_email-input\" class=\" \" name=\"em\" type=\"text\">\n",
      "<input type=\"hidden\" name=\"pn\" id=\"page_footer_iaFooter_emailSignup_elements_zWidget1-input\" value=\"Zazzle\">\n",
      "Sign me up!\n",
      "</form>\n",
      "========================================\n",
      "<form id=\"page_footer_iaFooter_emailSignup2-form\" class=\"AcAJ-form\" method=\"post\" action=\"http://www.zazzle.com/eml/signupdialog\" name=\"emailSignup2\" target=\"emailSignupWindow\" accept-charset=\"UTF-8\">\n",
      "<input id=\"page_footer_iaFooter_emailSignup2_elements_email-input\" class=\" \" name=\"em\" type=\"text\">\n",
      "<input type=\"hidden\" name=\"pn\" id=\"page_footer_iaFooter_emailSignup2_elements_zWidget1-input\" value=\"Zazzle\">\n",
      "</form>\n",
      "========================================\n",
      "<form id=\"page_footer_iaFooter_emailSignup-form\" class=\"AcAJ-form\" method=\"post\" action=\"https://www.zazzle.com/eml/signupdialog\" name=\"emailSignup\" target=\"emailSignupWindow\" accept-charset=\"UTF-8\">\n",
      "<input id=\"page_footer_iaFooter_emailSignup_elements_email-input\" class=\" \" name=\"em\" type=\"text\">\n",
      "<input type=\"hidden\" name=\"pn\" id=\"page_footer_iaFooter_emailSignup_elements_zWidget1-input\" value=\"Zazzle\">\n",
      "Sign me up!\n",
      "</form>\n",
      "========================================\n",
      "<form id=\"page_footer_iaFooter_emailSignup2-form\" class=\"AcAJ-form\" method=\"post\" action=\"https://www.zazzle.com/eml/signupdialog\" name=\"emailSignup2\" target=\"emailSignupWindow\" accept-charset=\"UTF-8\">\n",
      "<input id=\"page_footer_iaFooter_emailSignup2_elements_email-input\" class=\" \" name=\"em\" type=\"text\">\n",
      "<input type=\"hidden\" name=\"pn\" id=\"page_footer_iaFooter_emailSignup2_elements_zWidget1-input\" value=\"Zazzle\">\n",
      "</form>\n",
      "========================================\n"
     ]
    }
   ],
   "source": [
    "for form in mailing_list_forms:\n",
    "    tool.print_form_html(form)\n",
    "    print(\"=\"*40)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
