{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-12T01:55:25.334680Z",
     "start_time": "2019-07-12T01:55:24.809929Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from bs4 import BeautifulSoup\n",
    "# import urllib\n",
    "import urllib.request\n",
    "import codecs\n",
    "import os,glob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# EXTRACT TED Talk Names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-12T01:58:59.427126Z",
     "start_time": "2019-07-12T01:58:59.415650Z"
    }
   },
   "outputs": [],
   "source": [
    "def enlist_talk_names(path,dict_):\n",
    "    # r = urllib.urlopen(path).read()\n",
    "    r=urllib.request.urlopen(path).read()\n",
    "    soup = BeautifulSoup(r)\n",
    "    talks= soup.find_all(\"a\",class_='')\n",
    "    for i in talks:\n",
    "        if i.attrs['href'].find('/talks/')==0 and dict_.get(i.attrs['href'])!=1:\n",
    "            dict_[i.attrs['href']]=1\n",
    "    \n",
    "    return dict_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-12T02:00:27.898002Z",
     "start_time": "2019-07-12T02:00:12.610103Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/xiaofeiwu/miniconda3/envs/bigdata/lib/python3.6/site-packages/bs4/__init__.py:181: UserWarning: No parser was explicitly specified, so I'm using the best available HTML parser for this system (\"lxml\"). This usually isn't a problem, but if you run this code on another system, or in a different virtual environment, it may use a different parser and behave differently.\n",
      "\n",
      "The code that caused this warning is on line 193 of the file /Users/xiaofeiwu/miniconda3/envs/bigdata/lib/python3.6/runpy.py. To get rid of this warning, change code that looks like this:\n",
      "\n",
      " BeautifulSoup(YOUR_MARKUP})\n",
      "\n",
      "to this:\n",
      "\n",
      " BeautifulSoup(YOUR_MARKUP, \"lxml\")\n",
      "\n",
      "  markup_type=markup_type))\n"
     ]
    }
   ],
   "source": [
    "all_talk_names={}\n",
    "# for i in range(1,61):\n",
    "for i in range(1,10):\n",
    "    path='https://www.ted.com/talks?page=%d'%(i)\n",
    "    all_talk_names=enlist_talk_names(path,all_talk_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-12T02:01:50.474021Z",
     "start_time": "2019-07-12T02:01:50.436252Z"
    }
   },
   "outputs": [],
   "source": [
    "def extract_talk(path,talk_name):\n",
    "    # r=urllib.urlopen(path).read()\n",
    "    r=urllib.request.urlopen(path).read()\n",
    "    soup=BeautifulSoup(r)\n",
    "    df=pd.DataFrame()\n",
    "    print(path)\n",
    "    for i in soup.findAll('link'):\n",
    "        if i.get('href')!=None and i.attrs['href'].find('?language=')!=-1:\n",
    "            #print i.attrs['href']\n",
    "            lang=i.attrs['hreflang']\n",
    "            path=i.attrs['href']\n",
    "            # r1=urllib.urlopen(path).read()\n",
    "            r1=urllib.request.urlopen(path).read()\n",
    "            soup1=BeautifulSoup(r1)\n",
    "            time_frame=[]\n",
    "            text_talk=[]\n",
    "            for i in soup1.findAll('span',class_='talk-transcript__fragment'):\n",
    "                time_frame.append(i.attrs['data-time'])\n",
    "                text_talk.append(i.text.replace('\\n',' '))\n",
    "            #print len(time_frame),len(text_talk)\n",
    "            df1=pd.DataFrame()\n",
    "            df1[lang]=text_talk\n",
    "            df1[lang+'_time_frame']=time_frame\n",
    "            df=pd.concat([df,df1],axis=1)\n",
    "    df.to_csv(talk_name+'.csv',sep='\\t',encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-12T02:02:44.351126Z",
     "start_time": "2019-07-12T02:01:52.678803Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/xiaofeiwu/miniconda3/envs/bigdata/lib/python3.6/site-packages/bs4/__init__.py:181: UserWarning: No parser was explicitly specified, so I'm using the best available HTML parser for this system (\"lxml\"). This usually isn't a problem, but if you run this code on another system, or in a different virtual environment, it may use a different parser and behave differently.\n",
      "\n",
      "The code that caused this warning is on line 193 of the file /Users/xiaofeiwu/miniconda3/envs/bigdata/lib/python3.6/runpy.py. To get rid of this warning, change code that looks like this:\n",
      "\n",
      " BeautifulSoup(YOUR_MARKUP})\n",
      "\n",
      "to this:\n",
      "\n",
      " BeautifulSoup(YOUR_MARKUP, \"lxml\")\n",
      "\n",
      "  markup_type=markup_type))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://www.ted.com/talks/amy_padnani_how_we_re_honoring_people_overlooked_by_history/transcript\n",
      "https://www.ted.com/talks/jamie_paik_origami_robots_that_reshape_and_transform_themselves/transcript\n",
      "https://www.ted.com/talks/emma_bryce_what_is_hpv_and_how_can_you_protect_yourself_from_it/transcript\n",
      "https://www.ted.com/talks/rick_doblin_the_future_of_psychedelic_assisted_psychotherapy/transcript\n",
      "https://www.ted.com/talks/emma_lazarus_new_colossus/transcript\n",
      "https://www.ted.com/talks/barbara_j_king_grief_and_love_in_the_animal_kingdom/transcript\n",
      "https://www.ted.com/talks/natalie_fratto_3_ways_to_measure_your_adaptability_and_how_to_improve_it/transcript\n",
      "https://www.ted.com/talks/julius_maada_bio_a_vision_for_the_future_of_sierra_leone/transcript\n",
      "https://www.ted.com/talks/moriba_jah_the_worlds_first_crowdsourced_space_traffic_monitoring_system/transcript\n",
      "https://www.ted.com/talks/camille_defrenne_and_suzanne_simard_the_secret_language_of_trees/transcript\n"
     ]
    },
    {
     "ename": "HTTPError",
     "evalue": "HTTP Error 404: Not Found",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mHTTPError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-14-9b729b75d0e9>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mall_talk_names\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m     \u001b[0mextract_talk\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'https://www.ted.com'\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;34m'/transcript'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m7\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-13-081d68d14cf3>\u001b[0m in \u001b[0;36mextract_talk\u001b[0;34m(path, talk_name)\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mextract_talk\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mtalk_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m     \u001b[0;31m# r=urllib.urlopen(path).read()\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m     \u001b[0mr\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0murllib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrequest\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0murlopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m     \u001b[0msoup\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mBeautifulSoup\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0mdf\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDataFrame\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/bigdata/lib/python3.6/urllib/request.py\u001b[0m in \u001b[0;36murlopen\u001b[0;34m(url, data, timeout, cafile, capath, cadefault, context)\u001b[0m\n\u001b[1;32m    221\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    222\u001b[0m         \u001b[0mopener\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_opener\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 223\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mopener\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0murl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    224\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    225\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0minstall_opener\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mopener\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/bigdata/lib/python3.6/urllib/request.py\u001b[0m in \u001b[0;36mopen\u001b[0;34m(self, fullurl, data, timeout)\u001b[0m\n\u001b[1;32m    530\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mprocessor\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprocess_response\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprotocol\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    531\u001b[0m             \u001b[0mmeth\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprocessor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmeth_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 532\u001b[0;31m             \u001b[0mresponse\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmeth\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mreq\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    533\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    534\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/bigdata/lib/python3.6/urllib/request.py\u001b[0m in \u001b[0;36mhttp_response\u001b[0;34m(self, request, response)\u001b[0m\n\u001b[1;32m    640\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;36m200\u001b[0m \u001b[0;34m<=\u001b[0m \u001b[0mcode\u001b[0m \u001b[0;34m<\u001b[0m \u001b[0;36m300\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    641\u001b[0m             response = self.parent.error(\n\u001b[0;32m--> 642\u001b[0;31m                 'http', request, response, code, msg, hdrs)\n\u001b[0m\u001b[1;32m    643\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    644\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/bigdata/lib/python3.6/urllib/request.py\u001b[0m in \u001b[0;36merror\u001b[0;34m(self, proto, *args)\u001b[0m\n\u001b[1;32m    568\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mhttp_err\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    569\u001b[0m             \u001b[0margs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mdict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'default'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'http_error_default'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0morig_args\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 570\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_chain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    571\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    572\u001b[0m \u001b[0;31m# XXX probably also want an abstract factory that knows when it makes\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/bigdata/lib/python3.6/urllib/request.py\u001b[0m in \u001b[0;36m_call_chain\u001b[0;34m(self, chain, kind, meth_name, *args)\u001b[0m\n\u001b[1;32m    502\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhandler\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mhandlers\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    503\u001b[0m             \u001b[0mfunc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhandler\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmeth_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 504\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    505\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mresult\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    506\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/bigdata/lib/python3.6/urllib/request.py\u001b[0m in \u001b[0;36mhttp_error_default\u001b[0;34m(self, req, fp, code, msg, hdrs)\u001b[0m\n\u001b[1;32m    648\u001b[0m \u001b[0;32mclass\u001b[0m \u001b[0mHTTPDefaultErrorHandler\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mBaseHandler\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    649\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mhttp_error_default\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreq\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcode\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmsg\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhdrs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 650\u001b[0;31m         \u001b[0;32mraise\u001b[0m \u001b[0mHTTPError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mreq\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfull_url\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcode\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmsg\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhdrs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfp\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    651\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    652\u001b[0m \u001b[0;32mclass\u001b[0m \u001b[0mHTTPRedirectHandler\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mBaseHandler\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mHTTPError\u001b[0m: HTTP Error 404: Not Found"
     ]
    }
   ],
   "source": [
    "for i in all_talk_names:\n",
    "    extract_talk('https://www.ted.com'+i+'/transcript',i[7:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-12T02:04:44.710392Z",
     "start_time": "2019-07-12T02:04:24.500242Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "amy_padnani_how_we_re_honoring_people_overlooked_by_history.csv\n",
      "emma_bryce_what_is_hpv_and_how_can_you_protect_yourself_from_it.csv\n",
      "camille_defrenne_and_suzanne_simard_the_secret_language_of_trees.csv\n",
      "julius_maada_bio_a_vision_for_the_future_of_sierra_leone.csv\n",
      "jamie_paik_origami_robots_that_reshape_and_transform_themselves.csv\n",
      "moriba_jah_the_worlds_first_crowdsourced_space_traffic_monitoring_system.csv\n",
      "rick_doblin_the_future_of_psychedelic_assisted_psychotherapy.csv\n",
      "natalie_fratto_3_ways_to_measure_your_adaptability_and_how_to_improve_it.csv\n",
      "barbara_j_king_grief_and_love_in_the_animal_kingdom.csv\n",
      "emma_lazarus_new_colossus.csv\n"
     ]
    }
   ],
   "source": [
    "# path='/home/ajinkyak/Workspace/Image/Parse_talks/'\n",
    "# os.chdir(path)\n",
    "pieces=[]\n",
    "for file in glob.glob('*.csv'):\n",
    "    print(file)\n",
    "    frame=pd.read_csv(path+file,sep='\\t',encoding='utf-8')\n",
    "    pieces.append(frame)\n",
    "df= pd.concat(pieces, ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-12T02:04:48.929397Z",
     "start_time": "2019-07-12T02:04:48.864038Z"
    }
   },
   "outputs": [],
   "source": [
    "df.to_csv('All_TED_TALKS_DATA_1.csv',sep='\\t',encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-12T02:04:52.159952Z",
     "start_time": "2019-07-12T02:04:52.135277Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>&lt;!DOCTYPE html&gt;</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>&lt;!--[if lt IE 8]&gt; &lt;html class=\"no-js loggedout...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>&lt;!--[if IE 8]&gt; &lt;html class=\"no-js loggedout ol...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>&lt;!--[if gt IE 8]&gt;&lt;!--&gt;&lt;html class='no-js logge...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>&lt;head&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>&lt;script&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>(function (H){</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>H.className=H.className.replace(/\\bno-js\\b/,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>if (('; '+document.cookie).match(/; _ted_use...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>})(document.documentElement)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>&lt;/script&gt;&lt;meta charset='utf-8'&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>&lt;title&gt;TED Talks&lt;/title&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>&lt;meta name=\"description\" content=\"TED Talks ar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>&lt;meta name=\"rss-feed\" content=\"https://www.ted...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>&lt;meta name=\"keywords\" content=\"TED, Talks, The...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>&lt;link rel=\"mask-icon\" href=\"https://pa.tedcdn....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>&lt;meta name=\"theme-color\" content=\"#E62B1E\" /&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>&lt;link rel=\"shortcut icon\" href=\"https://pa.ted...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>&lt;meta name=\"HandheldFriendly\" content=\"True\" /&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>&lt;meta name=\"MobileOptimized\" content=\"320\" /&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>&lt;meta name=\"viewport\" content=\"width=device-wi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>&lt;meta name=\"apple-mobile-web-app-title\" conten...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>&lt;meta name=\"apple-mobile-web-app-capable\" cont...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>&lt;meta name=\"apple-mobile-web-app-status-bar-st...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>&lt;link rel=\"apple-touch-icon\" href=\"https://pa....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>&lt;link rel=\"apple-touch-icon-precomposed\" href=...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>&lt;meta name=\"application-name\" content=\"TED Tal...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>&lt;meta name=\"msapplication-config\" content=\"htt...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>&lt;meta name=\"msapplication-TileColor\" content=\"...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>&lt;meta http-equiv=\"cleartype\" content=\"on\" /&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>&lt;meta name=\"title\" content=\"TED Talks\" /&gt;&lt;!-- ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16372</th>\n",
       "      <td>&lt;/div&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16373</th>\n",
       "      <td>&lt;div id=\"activity-feed\" style=\"position: relat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16374</th>\n",
       "      <td>&lt;script&gt;g(\"activityFeed\",\"#activity-feed\",{})&lt;...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16375</th>\n",
       "      <td>&lt;script src='https://pa.tedcdn.com/javascripts...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16376</th>\n",
       "      <td>&lt;script async src='https://pa.tedcdn.com/javas...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16377</th>\n",
       "      <td>(function() {</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16378</th>\n",
       "      <td>var gads = document.createElement(\"script\");</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16379</th>\n",
       "      <td>gads.async = true;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16380</th>\n",
       "      <td>gads.type = \"text/javascript\";</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16381</th>\n",
       "      <td>gads.src = 'https://securepubads.g.doublec...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16382</th>\n",
       "      <td>var node = document.getElementsByTagName(\"...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16383</th>\n",
       "      <td>node.parentNode.insertBefore(gads, node);</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16384</th>\n",
       "      <td>})();</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16385</th>\n",
       "      <td>&lt;/script&gt;&lt;script&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16386</th>\n",
       "      <td>(function(s,o,b,a,m){</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16387</th>\n",
       "      <td>a=s.createElement(o),m=s.getElementsByTagN...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16388</th>\n",
       "      <td>a.async=1;a.src=b;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16389</th>\n",
       "      <td>m.parentNode.insertBefore(a,m)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16390</th>\n",
       "      <td>})(document,'script','https://www.google-ana...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16391</th>\n",
       "      <td>&lt;/script&gt;&lt;script&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16392</th>\n",
       "      <td>var _comscore = _comscore || [];</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16393</th>\n",
       "      <td>_comscore.push({ c1: \"2\", c2: \"7341760\" });</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16394</th>\n",
       "      <td>(function() {</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16395</th>\n",
       "      <td>var s = document.createElement(\"script\"), ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16396</th>\n",
       "      <td>s.src = (document.location.protocol == \"ht...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16397</th>\n",
       "      <td>el.parentNode.insertBefore(s, el);</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16398</th>\n",
       "      <td>})();</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16399</th>\n",
       "      <td>&lt;/script&gt;&lt;script&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16400</th>\n",
       "      <td>(function(d,s,u,e,m,r){r=(d.location.search|...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16401</th>\n",
       "      <td>&lt;/script&gt;&lt;/body&gt;&lt;/html&gt;</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>16402 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                         <!DOCTYPE html>\n",
       "0      <!--[if lt IE 8]> <html class=\"no-js loggedout...\n",
       "1      <!--[if IE 8]> <html class=\"no-js loggedout ol...\n",
       "2      <!--[if gt IE 8]><!--><html class='no-js logge...\n",
       "3                                                 <head>\n",
       "4                                               <script>\n",
       "5                                         (function (H){\n",
       "6        H.className=H.className.replace(/\\bno-js\\b/,...\n",
       "7        if (('; '+document.cookie).match(/; _ted_use...\n",
       "8                           })(document.documentElement)\n",
       "9                        </script><meta charset='utf-8'>\n",
       "10                              <title>TED Talks</title>\n",
       "11     <meta name=\"description\" content=\"TED Talks ar...\n",
       "12     <meta name=\"rss-feed\" content=\"https://www.ted...\n",
       "13     <meta name=\"keywords\" content=\"TED, Talks, The...\n",
       "14     <link rel=\"mask-icon\" href=\"https://pa.tedcdn....\n",
       "15         <meta name=\"theme-color\" content=\"#E62B1E\" />\n",
       "16     <link rel=\"shortcut icon\" href=\"https://pa.ted...\n",
       "17       <meta name=\"HandheldFriendly\" content=\"True\" />\n",
       "18         <meta name=\"MobileOptimized\" content=\"320\" />\n",
       "19     <meta name=\"viewport\" content=\"width=device-wi...\n",
       "20     <meta name=\"apple-mobile-web-app-title\" conten...\n",
       "21     <meta name=\"apple-mobile-web-app-capable\" cont...\n",
       "22     <meta name=\"apple-mobile-web-app-status-bar-st...\n",
       "23     <link rel=\"apple-touch-icon\" href=\"https://pa....\n",
       "24     <link rel=\"apple-touch-icon-precomposed\" href=...\n",
       "25     <meta name=\"application-name\" content=\"TED Tal...\n",
       "26     <meta name=\"msapplication-config\" content=\"htt...\n",
       "27     <meta name=\"msapplication-TileColor\" content=\"...\n",
       "28          <meta http-equiv=\"cleartype\" content=\"on\" />\n",
       "29     <meta name=\"title\" content=\"TED Talks\" /><!-- ...\n",
       "...                                                  ...\n",
       "16372                                             </div>\n",
       "16373  <div id=\"activity-feed\" style=\"position: relat...\n",
       "16374  <script>g(\"activityFeed\",\"#activity-feed\",{})<...\n",
       "16375  <script src='https://pa.tedcdn.com/javascripts...\n",
       "16376  <script async src='https://pa.tedcdn.com/javas...\n",
       "16377                                      (function() {\n",
       "16378       var gads = document.createElement(\"script\");\n",
       "16379                                 gads.async = true;\n",
       "16380                     gads.type = \"text/javascript\";\n",
       "16381      gads.src = 'https://securepubads.g.doublec...\n",
       "16382      var node = document.getElementsByTagName(\"...\n",
       "16383          node.parentNode.insertBefore(gads, node);\n",
       "16384                                              })();\n",
       "16385                                  </script><script>\n",
       "16386                              (function(s,o,b,a,m){\n",
       "16387      a=s.createElement(o),m=s.getElementsByTagN...\n",
       "16388                                 a.async=1;a.src=b;\n",
       "16389                     m.parentNode.insertBefore(a,m)\n",
       "16390    })(document,'script','https://www.google-ana...\n",
       "16391                                  </script><script>\n",
       "16392                   var _comscore = _comscore || [];\n",
       "16393        _comscore.push({ c1: \"2\", c2: \"7341760\" });\n",
       "16394                                      (function() {\n",
       "16395      var s = document.createElement(\"script\"), ...\n",
       "16396      s.src = (document.location.protocol == \"ht...\n",
       "16397                 el.parentNode.insertBefore(s, el);\n",
       "16398                                              })();\n",
       "16399                                  </script><script>\n",
       "16400    (function(d,s,u,e,m,r){r=(d.location.search|...\n",
       "16401                            </script></body></html>\n",
       "\n",
       "[16402 rows x 1 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[['ar','en','fr']]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
