import re
import unittest
import ir_datasets
from ir_datasets.datasets.clueweb09 import TrecWebTrackQuery, TrecPrel
from ir_datasets.formats import TrecQrel, TrecSubtopic, GenericDoc, GenericQuery, WarcDoc
from .base import DatasetIntegrationTest


_logger = ir_datasets.log.easy()


class TestClueWeb09(DatasetIntegrationTest):
    def test_clueweb09_docs(self):
        self._test_docs('clueweb09', items={
            0: WarcDoc('clueweb09-ar0000-00-00000', 'http://0098shop.com/product_"EH24_A\'1C3_Forex_(\'2\'131E\'�G.html', '2009-03-84T15:35:08-0700', re.compile(b'^HTTP/1\\.1 200 OK\nServer: Apache/2\\.2\\.11 \\(Unix\\) mod_ssl/2\\.2\\.11 OpenSSL/0\\.9\\.8b DAV/2 mod_auth_passthroug.{92}\nConnection: close\nContent\\-Type: text/html\nDate: Fri, 27 Feb 2009 16:04:39 GMT\nContent\\-Length: 38889$', flags=16), re.compile(b'^<meta httpequiv=Content\\-Type content="text/html; charset=utf\\-8"><meta httpequiv=Content\\-Type content.{38691}m3d\'\\);\\\r\n</SCRIPT>\\\r\n</span>\n\\\t<span id=\'HAM3D_counter\' class=\'HAM3D_hidden\'></span>\n</body>\n</html>\n\n\n$', flags=16), 'text/html'),
            9: WarcDoc('clueweb09-ar0000-00-00009', 'http://00perdomain.com/kids_and_teens/international/arabic/', '2009-03-84T15:35:08-0700', re.compile(b'^HTTP/1\\.1 200 OK\nContent\\-Type: text/html; charset=utf\\-8\nDate: Mon, 09 Feb 2009 12:41:10 GMT\nPragma: n.{145} sid=iorj3059uaka0isae61uh29494; path=/\nExpires: Thu, 19 Nov 1981 08:52:00 GMT\nContent\\-Length: 28444$', flags=16), re.compile(b'^\n\n\n<!\\-\\- Site by Zaz Corporation, 1 \\- 8 8 8 \\- 2 \\- Z A Z C O R  http://www\\.zazcorp\\.us \\-\\->\n\n\n\n<html><he.{28246}<script type="text/javascript">\n_uacct = "UA\\-488717\\-2";\nurchinTracker\\(\\);\n</script>\n\n</body></html>\n\n$', flags=16), 'text/html'),
            1000: WarcDoc('clueweb09-ar0000-00-01000', 'http://213.136.192.26/311276/ln59d.htm', '2009-03-84T15:35:13-0700', re.compile(b'^HTTP/1\\.1 200 OK\nAccept\\-Ranges: bytes\nContent\\-Type: text/html\nDate: Wed, 28 Jan 2009 18:47:22 GMT\nHos.{85}: close\nLast\\-Modified: Thu, 08 Jan 2009 00:28:36 GMT\nETag: "548bbb2871c91:a76"\nContent\\-Length: 65536$', flags=16), re.compile(b'^<html>\\\r\n  <head>\\\r\n    <META http\\-equiv="Content\\-Type" content="text/html; charset=windows\\-1256">\\\r\n  .{65338}\n                                                <font face="Simplified Arabic" color="\\#4D5064">\\\r\n\n\n$', flags=16), 'text/html'),
        })
        self._test_docs('clueweb09/ar', items={
            0: WarcDoc('clueweb09-ar0000-00-00000', 'http://0098shop.com/product_"EH24_A\'1C3_Forex_(\'2\'131E\'�G.html', '2009-03-84T15:35:08-0700', re.compile(b'^HTTP/1\\.1 200 OK\nServer: Apache/2\\.2\\.11 \\(Unix\\) mod_ssl/2\\.2\\.11 OpenSSL/0\\.9\\.8b DAV/2 mod_auth_passthroug.{92}\nConnection: close\nContent\\-Type: text/html\nDate: Fri, 27 Feb 2009 16:04:39 GMT\nContent\\-Length: 38889$', flags=16), re.compile(b'^<meta httpequiv=Content\\-Type content="text/html; charset=utf\\-8"><meta httpequiv=Content\\-Type content.{38691}m3d\'\\);\\\r\n</SCRIPT>\\\r\n</span>\n\\\t<span id=\'HAM3D_counter\' class=\'HAM3D_hidden\'></span>\n</body>\n</html>\n\n\n$', flags=16), 'text/html'),
            9: WarcDoc('clueweb09-ar0000-00-00009', 'http://00perdomain.com/kids_and_teens/international/arabic/', '2009-03-84T15:35:08-0700', re.compile(b'^HTTP/1\\.1 200 OK\nContent\\-Type: text/html; charset=utf\\-8\nDate: Mon, 09 Feb 2009 12:41:10 GMT\nPragma: n.{145} sid=iorj3059uaka0isae61uh29494; path=/\nExpires: Thu, 19 Nov 1981 08:52:00 GMT\nContent\\-Length: 28444$', flags=16), re.compile(b'^\n\n\n<!\\-\\- Site by Zaz Corporation, 1 \\- 8 8 8 \\- 2 \\- Z A Z C O R  http://www\\.zazcorp\\.us \\-\\->\n\n\n\n<html><he.{28246}<script type="text/javascript">\n_uacct = "UA\\-488717\\-2";\nurchinTracker\\(\\);\n</script>\n\n</body></html>\n\n$', flags=16), 'text/html'),
            1000: WarcDoc('clueweb09-ar0000-00-01000', 'http://213.136.192.26/311276/ln59d.htm', '2009-03-84T15:35:13-0700', re.compile(b'^HTTP/1\\.1 200 OK\nAccept\\-Ranges: bytes\nContent\\-Type: text/html\nDate: Wed, 28 Jan 2009 18:47:22 GMT\nHos.{85}: close\nLast\\-Modified: Thu, 08 Jan 2009 00:28:36 GMT\nETag: "548bbb2871c91:a76"\nContent\\-Length: 65536$', flags=16), re.compile(b'^<html>\\\r\n  <head>\\\r\n    <META http\\-equiv="Content\\-Type" content="text/html; charset=windows\\-1256">\\\r\n  .{65338}\n                                                <font face="Simplified Arabic" color="\\#4D5064">\\\r\n\n\n$', flags=16), 'text/html'),
        })
        self._test_docs('clueweb09/zh', items={
            0: WarcDoc('clueweb09-zh0000-00-00000', 'http://000027.istock.jrj.com.cn/', '2009-03-77T19:52:11-0700', re.compile(b'^HTTP/1\\.1 200 OK\nContent\\-Type: text/html; charset=GBK\nContent\\-Encoding: gzip\nVary: Accept\\-Encoding\nDa.{63}ose\nSet\\-Cookie: JSESSIONID=aWnxiSnSp6rg; path=/\nContent\\-Language: zh\\-CN, zh\\-CN\nContent\\-Length: 65536$', flags=16), re.compile(b'^\\\r\n\\\r\n\\\r\n\\\r\n\\\r\n\\\r\n<!DOCTYPE html PUBLIC "\\-//W3C//DTD XHTML 1\\.0 Transitional//EN" "http://www\\.w3\\.org/TR/xht.{65338}\\.com\\.cn/component/editor/imgs/B012\\.gif" height="30" width="30" onclick="oblog_InsertImg\\(this\\.src\\)"\n\n$', flags=16), 'text/html'),
            9: WarcDoc('clueweb09-zh0000-00-00009', 'http://000078.istock.jrj.com.cn/forum000078/topic1001345.html', '2009-03-77T19:52:11-0700', re.compile(b'^HTTP/1\\.1 200 OK\nContent\\-Type: text/html; charset=GBK\nContent\\-Encoding: gzip\nVary: Accept\\-Encoding\nDa.{63}ose\nSet\\-Cookie: JSESSIONID=bbPnYVFWx8D6; path=/\nContent\\-Language: zh\\-CN, zh\\-CN\nContent\\-Length: 26075$', flags=16), re.compile(b'^\\\r\n\\\r\n\\\r\n\\\r\n\\\r\n<!DOCTYPE html PUBLIC "\\-//W3C//DTD XHTML 1\\.0 Transitional//EN" "http://www\\.w3\\.org/TR/xhtml.{25877}lt="" />\\\r\n</body>\\\r\n<script src="http://istock\\.jrj\\.com\\.cn/includes/js/tongji\\.js"></script>\\\r\n</html>\n\n$', flags=16), 'text/html'),
            1000: WarcDoc('clueweb09-zh0000-00-01000', 'http://119go.com/shop/index.htm', '2009-03-77T19:52:15-0700', re.compile(b'^HTTP/1\\.1 200 OK\nContent\\-Type: text/html\nAccept\\-Ranges: bytes\nDate: Fri, 09 Jan 2009 21:54:09 GMT\nSer.{56} close\nLast\\-Modified: Fri, 19 Sep 2008 01:50:00 GMT\nETag: "054996fa19c91:337a"\nContent\\-Length: 18005$', flags=16), re.compile(b'^<!DOCTYPE HTML PUBLIC "\\-//W3C//DTD HTML 4\\.01 Transitional//EN"\\\r\n"http://www\\.w3\\.org/TR/html4/loose\\.dt.{17807}UA\\-105821\\-1";urchinTracker\\(\\);</script><!\\-\\-\\#include virtual="/include/ads\\.asp"\\-\\-></body>\\\r\n</html>\\\r\n\n\n$', flags=16), 'text/html'),
        })
        self._test_docs('clueweb09/en', items={
            0: WarcDoc('clueweb09-en0000-00-00000', 'http://00000-nrt-realestate.homepagestartup.com/', '2009-03-65T08:43:19-0800', re.compile(b'^HTTP/1\\.1 200 OK\nContent\\-Type: text/html\nDate: Tue, 13 Jan 2009 18:05:10 GMT\nPragma: no\\-cache\nCache\\-C.{100}Modified: Tue, 13 Jan 2009 18:05:10 GMT\nExpires: Mon, 20 Dec 1998 01:00:00 GMT\nContent\\-Length: 16254$', flags=16), re.compile(b'^<head> <meta http\\-equiv="Content\\-Language" content="en\\-gb"> <meta http\\-equiv="Content\\-Type" content=.{16056} 8pt">YouTube Videos</span></td> </tr> </table> </td> </tr> </table></div> </div> </body> </html> \n\n$', flags=16), 'text/html'),
            9: WarcDoc('clueweb09-en0000-00-00009', 'http://00perdomain.com/computers/', '2009-03-65T08:43:20-0800', re.compile(b'^HTTP/1\\.1 200 OK\nContent\\-Type: text/html; charset=utf\\-8\nDate: Sat, 17 Jan 2009 23:40:59 GMT\nPragma: n.{145} sid=i35idajmde65hlem4m2jpmrc37; path=/\nExpires: Thu, 19 Nov 1981 08:52:00 GMT\nContent\\-Length: 23500$', flags=16), re.compile(b'^\n\n\n<!\\-\\- Site by Zaz Corporation, 1 \\- 8 8 8 \\- 2 \\- Z A Z C O R  http://www\\.zazcorp\\.us \\-\\->\n\n\n\n<html><he.{23302}<script type="text/javascript">\n_uacct = "UA\\-488717\\-2";\nurchinTracker\\(\\);\n</script>\n\n</body></html>\n\n$', flags=16), 'text/html'),
            1000: WarcDoc('clueweb09-en0000-00-01000', 'http://2modern.com/designer/FLOS/Flos-Archimoon-Soft-Table-Lamp', '2009-03-65T08:44:07-0800', re.compile(b'^HTTP/1\\.1 200 OK\nContent\\-Type: text/html; charset=utf\\-8\nKeep\\-Alive: timeout=15, max=965\nContent\\-Encod.{359}4Pa38Ta38Nb350; path=/\nLast\\-Modified: Tue, 13 Jan 2009 21:10:47 GMT\nExpires: 0\nContent\\-Length: 52741$', flags=16), re.compile(b'^\n<html>\n<head>\n<meta http\\-equiv="Content\\-Type" content="text/html; charset=UTF\\-8">\n<title>FLOS \\- Arc.{52543}\\- \\[ 418126 \\] \\[  \\] \\[ /s\\.nl \\] \\[ Tue Jan 13 13:10:47 PST 2009 \\] \\-\\->\n<!\\-\\- Not logging slowest SQL \\-\\->\n\n\n$', flags=16), 'text/html'),
        })
        self._test_docs('clueweb09/fr', items={
            0: WarcDoc('clueweb09-fr0000-00-00000', 'http://0-charmedgallery.skyrock.com/', '2009-03-85T01:34:33-0700', re.compile(b'^HTTP/1\\.1 200 OK\nContent\\-Type: text/html\nContent\\-Encoding: gzip\nP3P: CP="NOI DSP COR CURa DEVa PSAa O.{328}xpires: Sun, 08 Feb 2009 15:13:41 GMT\nETag: "92c422f2ea325185cfadb9a9088b2b1f"\nContent\\-Length: 53898$', flags=16), re.compile(b'^<!DOCTYPE html PUBLIC "\\-//W3C//DTD XHTML 1\\.0 Strict//EN" "http://www\\.w3\\.org/TR/xhtml1/DTD/xhtml1\\-str.{53700}ascript" src="http://pagead2\\.googlesyndication\\.com/pagead/show_ads\\.js"></script>\n\n\n</body>\n</html>\n\n$', flags=16), 'text/html'),
            9: WarcDoc('clueweb09-fr0000-00-00009', 'http://000221.skyrock.com/', '2009-03-85T01:34:33-0700', re.compile(b'^HTTP/1\\.1 200 OK\nContent\\-Type: text/html\nContent\\-Encoding: gzip\nP3P: CP="NOI DSP COR CURa DEVa PSAa O.{329}xpires: Fri, 13 Feb 2009 21:03:18 GMT\nETag: "bc2011cb3798d5b21d8df1709d7270f1"\nContent\\-Length: 31594$', flags=16), re.compile(b'^<!DOCTYPE html PUBLIC "\\-//W3C//DTD XHTML 1\\.0 Strict//EN" "http://www\\.w3\\.org/TR/xhtml1/DTD/xhtml1\\-str.{31396}ascript" src="http://pagead2\\.googlesyndication\\.com/pagead/show_ads\\.js"></script>\n\n\n</body>\n</html>\n\n$', flags=16), 'text/html'),
            1000: WarcDoc('clueweb09-fr0000-00-01000', 'http://123maigrir.com.ivchost3.com/pratique/institut/institut2.htm', '2009-03-85T01:34:39-0700', re.compile(b'^HTTP/1\\.1 200 OK\nContent\\-Type: text/html\nAccept\\-Ranges: bytes\nDate: Thu, 29 Jan 2009 16:25:21 GMT\nSer.{95}ose\nLast\\-Modified: Fri, 15 Jun 2007 23:09:10 GMT\nETag: "18441c9\\-adbe\\-46731c16"\nContent\\-Length: 44478$', flags=16), re.compile(b'^<html><!\\-\\- \\#BeginTemplate "/Templates/page_menu_complet\\.dwt" \\-\\-><!\\-\\- DW6 \\-\\->\\\r\n<head>\\\r\n<script langua.{44280}"279" align="center">\\&nbsp;</td>\\\r\n  </tr>\\\r\n</table>\\\r\n\\\r\n</body>\\\r\n\\\r\n<!\\-\\- \\#EndTemplate \\-\\-></html>\\\r\n\\\r\n\n\n$', flags=16), 'text/html'),
        })
        self._test_docs('clueweb09/de', items={
            0: WarcDoc('clueweb09-de0000-00-00000', 'http://00perdomain.com/kids_and_teens/international/deutsch/', '2009-03-84T23:38:37-0700', re.compile(b'^HTTP/1\\.1 200 OK\nContent\\-Type: text/html; charset=utf\\-8\nDate: Sun, 08 Feb 2009 06:17:05 GMT\nPragma: n.{144}: sid=g3njt512js43a04j8vekkt0627; path=/\nExpires: Thu, 19 Nov 1981 08:52:00 GMT\nContent\\-Length: 9942$', flags=16), re.compile(b'^\n\n\n<!\\-\\- Site by Zaz Corporation, 1 \\- 8 8 8 \\- 2 \\- Z A Z C O R  http://www\\.zazcorp\\.us \\-\\->\n\n\n\n<html><he.{9744}<script type="text/javascript">\n_uacct = "UA\\-488717\\-2";\nurchinTracker\\(\\);\n</script>\n\n</body></html>\n\n$', flags=16), 'text/html'),
            9: WarcDoc('clueweb09-de0000-00-00009', 'http://00perdomain.com/world/deutsch/computer/', '2009-03-84T23:38:37-0700', re.compile(b'^HTTP/1\\.1 200 OK\nContent\\-Type: text/html; charset=utf\\-8\nDate: Tue, 03 Feb 2009 03:48:40 GMT\nCache\\-Con.{145} sid=fit0t220fh9vpsefik8l628866; path=/\nExpires: Thu, 19 Nov 1981 08:52:00 GMT\nContent\\-Length: 21957$', flags=16), re.compile(b'^\n\n\n<!\\-\\- Site by Zaz Corporation, 1 \\- 8 8 8 \\- 2 \\- Z A Z C O R  http://www\\.zazcorp\\.us \\-\\->\n\n\n\n<html><he.{21759}<script type="text/javascript">\n_uacct = "UA\\-488717\\-2";\nurchinTracker\\(\\);\n</script>\n\n</body></html>\n\n$', flags=16), 'text/html'),
            1000: WarcDoc('clueweb09-de0000-00-01000', 'http://1689494.rc-welt.com/', '2009-03-84T23:38:43-0700', re.compile(b'^HTTP/1\\.1 200 OK\nContent\\-Type: text/html; charset=iso\\-8859\\-1\nContent\\-Encoding: gzip\nDate: Mon, 26 Jan.{332}Modified: Mon, 26 Jan 2009 01:42:05 GMT\nExpires: Mon, 26 Jan 2009 01:42:05 GMT\nContent\\-Length: 38206$', flags=16), re.compile(b'^ <!DOCTYPE HTML PUBLIC "\\-//W3C//DTD HTML 4\\.01 Transitional//EN"> <html>  <head> <meta http\\-equiv="Co.{38008}"0" height="0" scrolling="no" name="dynamicFrame" src="RCheartbeat\\.php" ></iframe> </body> </html>\n\n$', flags=16), 'text/html'),
        })
        self._test_docs('clueweb09/it', items={
            0: WarcDoc('clueweb09-it0000-00-00000', 'http://00lucianoligabue00.giovani.it/', '2009-03-84T17:45:02-0700', re.compile(b'^HTTP/1\\.1 200 OK\nContent\\-Type: text/html\nContent\\-Encoding: gzip\nVary: Host,Accept\\-Encoding,User\\-Agent.{248}ec0cfa05ca1; path=/; domain=\\.giovani\\.it\nExpires: Thu, 19 Nov 1981 08:52:00 GMT\nContent\\-Length: 12895$', flags=16), re.compile(b'^<!DOCTYPE html PUBLIC "\\-//W3C//DTD XHTML 1\\.0 Transitional//EN" "http://www\\.w3\\.org/TR/xhtml1/DTD/xhtm.{12697}avascript">\n_uacct = "UA\\-746038\\-2";\n_udn="giovani\\.it";\nurchinTracker\\(\\);\n</script>\n</body>\n</html>\n\n\n$', flags=16), 'text/html'),
            9: WarcDoc('clueweb09-it0000-00-00009', 'http://00perdomain.com/world/furlan/sal%c3%bbt/', '2009-03-84T17:45:02-0700', re.compile(b'^HTTP/1\\.1 200 OK\nContent\\-Type: text/html; charset=utf\\-8\nDate: Sun, 08 Feb 2009 06:13:46 GMT\nCache\\-Con.{144}: sid=bqreqd4mrjrm3fi30405ldfnq6; path=/\nExpires: Thu, 19 Nov 1981 08:52:00 GMT\nContent\\-Length: 9794$', flags=16), re.compile(b'^\n\n\n<!\\-\\- Site by Zaz Corporation, 1 \\- 8 8 8 \\- 2 \\- Z A Z C O R  http://www\\.zazcorp\\.us \\-\\->\n\n\n\n<html><he.{9596}<script type="text/javascript">\n_uacct = "UA\\-488717\\-2";\nurchinTracker\\(\\);\n</script>\n\n</body></html>\n\n$', flags=16), 'text/html'),
            1000: WarcDoc('clueweb09-it0000-00-01000', 'http://aaronsummers.ifrance.com/new/socci/', '2009-03-84T17:45:07-0700', re.compile(b'^HTTP/1\\.1 200 OK\nContent\\-Type: text/html; charset=ISO\\-8859\\-1\nAccept\\-Ranges: bytes\nDate: Tue, 10 Feb 2.{70}lose\nLast\\-Modified: Wed, 10 Oct 2007 19:43:32 GMT\nETag: "804131\\-3d59\\-b356a100"\nContent\\-Length: 16483$', flags=16), re.compile(b'^<script language="Javascript" type="text/javascript">\n<!\\-\\-\nvar d=new Date; rnd=d\\.getDay\\(\\)\\+\'\\-\'\\+d\\.getH.{16285}\'<sc\'\\+\'ript src="http://js\\-perso\\.ifrance\\.com/js2\\.php\\?\'\\+rnd\\+\'"><\'\\+\'/sc\'\\+\'ript>\'\\);\n// \\-\\->\n</script>\n\n\n$', flags=16), 'text/html'),
        })
        self._test_docs('clueweb09/ja', items={
            0: WarcDoc('clueweb09-ja0000-00-00000', 'http://00077.web.fc2.com/', '2009-03-85T02:09:10-0700', re.compile(b'^HTTP/1\\.1 200 OK\nAccept\\-Ranges: bytes\nContent\\-Type: text/html\nDate: Tue, 13 Jan 2009 21:29:20 GMT\nSer.{98}ection: close\nLast\\-Modified: Tue, 18 Nov 2008 14:07:53 GMT\nContent\\-Language: en\nContent\\-Length: 9911$', flags=16), re.compile(b'^<!DOCTYPE html PUBLIC "\\-//W3C//DTD XHTML 1\\.0 Transitional//EN"\n "http://www\\.w3\\.org/TR/xhtml1/DTD/xht.{9713}_cs \\+ \'\\&dm=\' \\+ fhp_dm \\+ \'"><\' \\+ \'/script>\';\ndocument\\.write\\(fhp_wt\\);\n//\\-\\-></script></body>\n</html>\n\n\n$', flags=16), 'text/html'),
            9: WarcDoc('clueweb09-ja0000-00-00009', 'http://0095.jp/', '2009-03-85T02:09:10-0700', b'HTTP/1.1 200 OK\nContent-Type: text/html\nDate: Thu, 08 Jan 2009 04:28:30 GMT\nServer: Apache/1.3.39 (Unix)\nX-Powered-By: PHP/5.2.8\nConnection: close\nContent-Length: 20822', re.compile(b'^<!DOCTYPE html PUBLIC "\\-//W3C//DTD XHTML 1\\.0 Transitional//EN" "http://www\\.w3\\.org/TR/xhtml1/DTD/xhtm.{20624}r>\n\xa1\xa1\xa1\xa1\xa1\xa1\xa1\xa1\\&copy2009\\&nbsp;<a href="http://0095\\.jp/">\xa5\xad\xa5\xa6\xa5\xa4\xb6\xe6\xb3\xda\xc9\xf4</a></div>\n</div>\n</body>\n</html>\n\n\n$', flags=16), 'text/html'),
            1000: WarcDoc('clueweb09-ja0000-00-01000', 'http://1bunting.info/sanc3/0066/020/', '2009-03-85T02:09:15-0700', re.compile(b'^HTTP/1\\.1 200 OK\nAccept\\-Ranges: bytes\nContent\\-Type: text/html\nDate: Wed, 07 Jan 2009 12:39:57 GMT\nSer.{145}ast\\-Modified: Sun, 04 Jan 2009 02:38:48 GMT\nETag: "64d86f0\\-2d98\\-45f9f12e0be00"\nContent\\-Length: 11672$', flags=16), re.compile(b'^<!DOCTYPE html PUBLIC "\\-//W3C//DTD XHTML 1\\.0 Transitional//EN" "http://www\\.w3\\.org/TR/xhtml1/DTD/xhtm.{11474}en\\.height\\+"\\&"\\);\ndocument\\.write\\("color="\\+screen\\.colorDepth\\+"\'>"\\);\n// \\-\\->\n</SCRIPT>\n</body>\n</html>\n\n\n$', flags=16), 'text/html'),
        })
        self._test_docs('clueweb09/ko', items={
            0: WarcDoc('clueweb09-ko0000-00-00000', 'http://00perdomain.com/kids_and_teens/international/korean/', '2009-03-84T16:00:51-0700', re.compile(b'^HTTP/1\\.1 200 OK\nContent\\-Type: text/html; charset=utf\\-8\nDate: Wed, 11 Feb 2009 16:27:15 GMT\nPragma: n.{145} sid=6eoehtb6uqtbtc5i6j1ohsht55; path=/\nExpires: Thu, 19 Nov 1981 08:52:00 GMT\nContent\\-Length: 10481$', flags=16), re.compile(b'^\n\n\n<!\\-\\- Site by Zaz Corporation, 1 \\- 8 8 8 \\- 2 \\- Z A Z C O R  http://www\\.zazcorp\\.us \\-\\->\n\n\n\n<html><he.{10283}<script type="text/javascript">\n_uacct = "UA\\-488717\\-2";\nurchinTracker\\(\\);\n</script>\n\n</body></html>\n\n$', flags=16), 'text/html'),
            9: WarcDoc('clueweb09-ko0000-00-00009', 'http://00perdomain.com/world/korean/%ea%b1%b4%ea%b0%95,%ec%9d%98%ed%95%99/%ec%a7%88%eb%b3%91,%ec%a7%88%ed%99%98/', '2009-03-84T16:00:51-0700', re.compile(b'^HTTP/1\\.1 200 OK\nContent\\-Type: text/html; charset=utf\\-8\nDate: Fri, 27 Feb 2009 20:07:39 GMT\nCache\\-Con.{145} sid=6s444g9ae4nv9uk4mfbb9e2dv3; path=/\nExpires: Thu, 19 Nov 1981 08:52:00 GMT\nContent\\-Length: 18781$', flags=16), re.compile(b'^\n\n\n<!\\-\\- Site by Zaz Corporation, 1 \\- 8 8 8 \\- 2 \\- Z A Z C O R  http://www\\.zazcorp\\.us \\-\\->\n\n\n\n<html><he.{18583}<script type="text/javascript">\n_uacct = "UA\\-488717\\-2";\nurchinTracker\\(\\);\n</script>\n\n</body></html>\n\n$', flags=16), 'text/html'),
            1000: WarcDoc('clueweb09-ko0000-00-01000', 'http://208.70.77.133/9ea771f89c09d26fdd4f9309023b.html', '2009-03-84T16:00:55-0700', b'HTTP/1.1 200 OK\nContent-Type: text/html; charset=EUC-KR\nDate: Sat, 17 Jan 2009 08:48:54 GMT\nServer: Apache/2.0.52 (CentOS)\nX-Powered-By: PHP/5.1.6\nConnection: close\nContent-Length: 14595', re.compile(b'^<!DOCTYPE HTML PUBLIC "\\-//W3C//DTD HTML 4\\.0 Transitional//EN">\n<html>\n<head>\n<title>\xc1\xea\xbd\xc3\xc6\xae\xb7\xb9\xc0\xcc\xb4\xd7\xba\xb9 \xc1.{14397}:<a href="mailto:webmaster@208\\.70\\.77\\.133">\xc0\xa5\xb8\xb6\xbd\xba\xc5\xcd</a>\n</center>\n<body>\n</html>\n<!\\-\\- 0\\.110781 \\-\\->\n\n\n$', flags=16), 'text/html'),
        })
        self._test_docs('clueweb09/pt', items={
            0: WarcDoc('clueweb09-pt0000-00-00000', 'http://005bom.ezdir.net/', '2009-03-84T20:11:48-0700', re.compile(b'^HTTP/1\\.1 200 OK\nServer: Apache/1\\.3\\.37 \\(Unix\\) mod_auth_passthrough/1\\.8 mod_log_bytes/1\\.2 mod_bwlimite.{91}Content\\-Type: text/html; charset=iso\\-8859\\-1\nDate: Wed, 18 Feb 2009 23:32:21 GMT\nContent\\-Length: 1970$', flags=16), re.compile(b'^<HTML>\n<HEAD>\n<META NAME="Generator" CONTENT="ezDIR \\- www\\.ezdir\\.net">\n<META NAME="Description" CONTE.{1772}/ezdir\\.inweb\\.adm\\.br/ezdir/anuncie\\.php">Clique aqui</A></FONT>\n</TD></TR>\n</TABLE>\n</BODY>\n</HTML>\n\n\n$', flags=16), 'text/html'),
            9: WarcDoc('clueweb09-pt0000-00-00009', 'http://00perdomain.com/world/portugu%c3%aas/artes/artesanato/', '2009-03-84T20:11:48-0700', re.compile(b'^HTTP/1\\.1 200 OK\nContent\\-Type: text/html; charset=utf\\-8\nDate: Thu, 12 Feb 2009 05:26:23 GMT\nCache\\-Con.{145} sid=8j75130i6a5iibrmaq85ji1753; path=/\nExpires: Thu, 19 Nov 1981 08:52:00 GMT\nContent\\-Length: 18361$', flags=16), re.compile(b'^\n\n\n<!\\-\\- Site by Zaz Corporation, 1 \\- 8 8 8 \\- 2 \\- Z A Z C O R  http://www\\.zazcorp\\.us \\-\\->\n\n\n\n<html><he.{18163}<script type="text/javascript">\n_uacct = "UA\\-488717\\-2";\nurchinTracker\\(\\);\n</script>\n\n</body></html>\n\n$', flags=16), 'text/html'),
            1000: WarcDoc('clueweb09-pt0000-00-01000', 'http://1001gatos.org/john-wood-saiu-da-microsoft-para-mudar-o-mundo/', '2009-03-84T20:11:55-0700', re.compile(b'^HTTP/1\\.1 200 OK\nContent\\-Type: text/html; charset=UTF\\-8\nDate: Sun, 08 Feb 2009 00:43:44 GMT\nServer: A.{223}, 08\\-Feb\\-2010 00:43:45 GMT; path=/\nX\\-Pingback: http://1001gatos\\.org/xmlrpc\\.php\nContent\\-Length: 40352$', flags=16), re.compile(b'^<!DOCTYPE html PUBLIC "\\-//W3C//DTD XHTML 1\\.0 Strict//EN" "http://www\\.w3\\.org/TR/xhtml1/DTD/xhtml1\\-str.{40154}DynaboxConfig\\?div_nome=dynabox\\&amp;site_origem=8662542\\&amp;cor=cc0000"></script>\\\r\n</body>\\\r\n</html>\n\n$', flags=16), 'text/html'),
        })
        self._test_docs('clueweb09/es', items={
            0: WarcDoc('clueweb09-es0000-00-00000', 'http://00001101.blogspot.com/', '2009-03-81T23:59:24-0700', re.compile(b'^HTTP/1\\.1 200 OK\nContent\\-Type: text/html; charset=UTF\\-8\nDate: Sat, 24 Jan 2009 18:07:26 GMT\nCache\\-Con.{101}ed: Sat, 17 Jan 2009 18:35:52 GMT\nETag: "a341615f\\-6594\\-4524\\-ae74\\-53101f87cdba"\nContent\\-Length: 34361$', flags=16), re.compile(b'^<!DOCTYPE html PUBLIC "\\-//W3C//DTD XHTML 1\\.0 Strict//EN" "http://www\\.w3\\.org/TR/xhtml1/DTD/xhtml1\\-str.{34163}\\.com/fb/images/pub/feed\\-icon16x16\\.png" alt="" style="border:0"/></a></p>\\\r\n</div>\\\r\n</body>\\\r\n</html>\n\n$', flags=16), 'text/html'),
            9: WarcDoc('clueweb09-es0000-00-00009', 'http://00perdomain.com/world/espa%c3%b1ol/', '2009-03-81T23:59:24-0700', re.compile(b'^HTTP/1\\.1 200 OK\nContent\\-Type: text/html; charset=utf\\-8\nDate: Thu, 29 Jan 2009 19:26:46 GMT\nCache\\-Con.{144}: sid=javrlcp0o652co0p1gvpttlqe6; path=/\nExpires: Thu, 19 Nov 1981 08:52:00 GMT\nContent\\-Length: 9073$', flags=16), re.compile(b'^\n\n\n<!\\-\\- Site by Zaz Corporation, 1 \\- 8 8 8 \\- 2 \\- Z A Z C O R  http://www\\.zazcorp\\.us \\-\\->\n\n\n\n<html><he.{8875}<script type="text/javascript">\n_uacct = "UA\\-488717\\-2";\nurchinTracker\\(\\);\n</script>\n\n</body></html>\n\n$', flags=16), 'text/html'),
            1000: WarcDoc('clueweb09-es0000-00-01000', 'http://abrahamzabludovsky.radiotrece.com.mx/2008/11/27/deportes-del-27-de-noviembre-de-2008/', '2009-03-81T23:59:31-0700', re.compile(b'^HTTP/1\\.1 200 OK\nContent\\-Type: text/html; charset=UTF\\-8\nDate: Mon, 09 Feb 2009 08:53:49 GMT\nServer: A.{51}tion: close\nX\\-Pingback: http://abrahamzabludovsky\\.radiotrece\\.com\\.mx/xmlrpc\\.php\nContent\\-Length: 28172$', flags=16), re.compile(b'^<!DOCTYPE html PUBLIC "\\-//W3C//DTD XHTML 1\\.0 Strict//EN" "http://www\\.w3\\.org/TR/xhtml1/DTD/xhtml1\\-str.{27974}UA\\-1620189\\-24"\\);\npageTracker\\._initData\\(\\);\npageTracker\\._trackPageview\\(\\);\n</script>\n</body>\n</html>\n\n\n$', flags=16), 'text/html'),
        })
        self._test_docs('clueweb09/catb', items={
            0: WarcDoc('clueweb09-en0000-00-00000', 'http://00000-nrt-realestate.homepagestartup.com/', '2009-03-65T08:43:19-0800', re.compile(b'^HTTP/1\\.1 200 OK\nContent\\-Type: text/html\nDate: Tue, 13 Jan 2009 18:05:10 GMT\nPragma: no\\-cache\nCache\\-C.{100}Modified: Tue, 13 Jan 2009 18:05:10 GMT\nExpires: Mon, 20 Dec 1998 01:00:00 GMT\nContent\\-Length: 16254$', flags=16), re.compile(b'^<head> <meta http\\-equiv="Content\\-Language" content="en\\-gb"> <meta http\\-equiv="Content\\-Type" content=.{16056} 8pt">YouTube Videos</span></td> </tr> </table> </td> </tr> </table></div> </div> </body> </html> \n\n$', flags=16), 'text/html'),
            9: WarcDoc('clueweb09-en0000-00-00009', 'http://00perdomain.com/computers/', '2009-03-65T08:43:20-0800', re.compile(b'^HTTP/1\\.1 200 OK\nContent\\-Type: text/html; charset=utf\\-8\nDate: Sat, 17 Jan 2009 23:40:59 GMT\nPragma: n.{145} sid=i35idajmde65hlem4m2jpmrc37; path=/\nExpires: Thu, 19 Nov 1981 08:52:00 GMT\nContent\\-Length: 23500$', flags=16), re.compile(b'^\n\n\n<!\\-\\- Site by Zaz Corporation, 1 \\- 8 8 8 \\- 2 \\- Z A Z C O R  http://www\\.zazcorp\\.us \\-\\->\n\n\n\n<html><he.{23302}<script type="text/javascript">\n_uacct = "UA\\-488717\\-2";\nurchinTracker\\(\\);\n</script>\n\n</body></html>\n\n$', flags=16), 'text/html'),
            1000: WarcDoc('clueweb09-en0000-00-01000', 'http://2modern.com/designer/FLOS/Flos-Archimoon-Soft-Table-Lamp', '2009-03-65T08:44:07-0800', re.compile(b'^HTTP/1\\.1 200 OK\nContent\\-Type: text/html; charset=utf\\-8\nKeep\\-Alive: timeout=15, max=965\nContent\\-Encod.{359}4Pa38Ta38Nb350; path=/\nLast\\-Modified: Tue, 13 Jan 2009 21:10:47 GMT\nExpires: 0\nContent\\-Length: 52741$', flags=16), re.compile(b'^\n<html>\n<head>\n<meta http\\-equiv="Content\\-Type" content="text/html; charset=UTF\\-8">\n<title>FLOS \\- Arc.{52543}\\- \\[ 418126 \\] \\[  \\] \\[ /s\\.nl \\] \\[ Tue Jan 13 13:10:47 PST 2009 \\] \\-\\->\n<!\\-\\- Not logging slowest SQL \\-\\->\n\n\n$', flags=16), 'text/html'),
        })
        self._assert_namedtuple(ir_datasets.load('clueweb09').docs.lookup('clueweb09-en0007-01-40637'), WarcDoc('clueweb09-en0007-01-40637', 'http://www.job-hunt.org/marketing.shtml', '2009-03-65T12:44:49-0800', re.compile(b'HTTP/1.1 200 OK.*44437', flags=16), body=re.compile(b'<!DOCTYPE HTML .*</script> \n\n</BODY>\n\n</HTML>\n\n\n', flags=16), body_content_type='text/html'))

    def test_clueweb09_docstore(self):
        docstore = ir_datasets.load('clueweb09').docs_store()
        docstore.clear_cache()
        with _logger.duration('cold fetch'):
            result = docstore.get_many(['clueweb09-en0000-00-00003', 'clueweb09-en0000-00-35154', 'clueweb09-ar0000-48-02342'])
            self.assertEqual(len(result), 3)
        with _logger.duration('warm fetch'):
            result = docstore.get_many(['clueweb09-en0000-00-00003', 'clueweb09-en0000-00-35154', 'clueweb09-ar0000-48-02342'])
            self.assertEqual(len(result), 3)
        docstore = ir_datasets.load('clueweb09').docs_store()
        with _logger.duration('warm fetch (new docstore)'):
            result = docstore.get_many(['clueweb09-en0000-00-00003', 'clueweb09-en0000-00-35154', 'clueweb09-ar0000-48-02342'])
            self.assertEqual(len(result), 3)
        with _logger.duration('cold fetch (nearby)'):
            result = docstore.get_many(['clueweb09-en0000-00-00023', 'clueweb09-en0000-00-35167', 'clueweb09-ar0000-48-02348'])
            self.assertEqual(len(result), 3)
        with _logger.duration('cold fetch (earlier)'):
            result = docstore.get_many(['clueweb09-en0000-00-00001', 'clueweb09-ar0000-48-00009'])
            self.assertEqual(len(result), 2)


    def test_clueweb09_queries(self):
        self._test_queries('clueweb09/en/trec-web-2009', count=50, items={
            0: TrecWebTrackQuery('1', 'obama family tree', "Find information on President Barack Obama's family\n  history, including genealogy, national origins, places and dates of\n  birth, etc.\n  ", 'faceted', (TrecSubtopic(number='1', text='\n    Find the TIME magazine photo essay "Barack Obama\'s Family Tree".\n  ', type='nav'), TrecSubtopic(number='2', text="\n    Where did Barack Obama's parents and grandparents come from?\n  ", type='inf'), TrecSubtopic(number='3', text="\n    Find biographical information on Barack Obama's mother.\n  ", type='inf'))),
            9: TrecWebTrackQuery('10', 'cheap internet', "I'm looking for cheap (i.e. low-cost) internet service.\n  ", 'faceted', (TrecSubtopic(number='1', text='\n    What are some low-cost broadband internet providers?\n  ', type='inf'), TrecSubtopic(number='2', text='\n    Do any internet providers still sell dial-up?\n  ', type='inf'), TrecSubtopic(number='3', text='\n    Who can provide inexpensive digital cable television bundled with\n    internet service?\n  ', type='inf'), TrecSubtopic(number='4', text="\n    I'm looking for the Vonage homepage.\n  ", type='nav'), TrecSubtopic(number='5', text='\n    Find me some providers of free wireless internet access.\n  ', type='inf'), TrecSubtopic(number='6', text='\n    I want to find cheap DSL providers.\n  ', type='inf'), TrecSubtopic(number='7', text='\n    Is there a way to get internet access without phone service?\n  ', type='inf'), TrecSubtopic(number='8', text="\n    Take me to Comcast's homepage.\n  ", type='nav'))),
            49: TrecWebTrackQuery('50', 'dog heat', 'What is the effect of excessive heat on dogs?\n  ', 'ambiguous', (TrecSubtopic(number='1', text='\n    What is the effect of excessive heat on dogs?\n  ', type='inf'), TrecSubtopic(number='2', text='\n    What are symptoms of heat stroke and other heat-related illnesses\n    in dogs?\n  ', type='inf'), TrecSubtopic(number='3', text='\n    Find information on dogs\' reproductive cycle.  What does it mean\n    when a dog is "in heat"?\n  ', type='inf'))),
        })
        self._test_queries('clueweb09/en/trec-web-2010', count=50, items={
            0: TrecWebTrackQuery('51', 'horse hooves', '\n    Find information about horse hooves, their care, and diseases of hooves.\n  ', 'faceted', (TrecSubtopic(number='1', text="\n    Find information about horses' hooves and how to care for them.\n  ", type='inf'), TrecSubtopic(number='2', text='\n    Find pictures of horse hooves.\n  ', type='nav'), TrecSubtopic(number='3', text='\n    What are some injuries or diseases of hooves in horses, and how\n    are they treated?\n  ', type='inf'), TrecSubtopic(number='4', text="\n    Describe the anatomy of horses' feet and hooves.\n  ", type='inf'), TrecSubtopic(number='5', text='\n    Find information on shoeing horses and horseshoe problems.\n  ', type='inf'))),
            9: TrecWebTrackQuery('60', 'bellevue', '\n    Find information about Bellevue, Washington.\n  ', 'ambiguous', (TrecSubtopic(number='1', text='\n    Find information about Bellevue, Washington.\n  ', type='inf'), TrecSubtopic(number='2', text='\n    Find information about Bellevue, Nebraska.\n  ', type='inf'), TrecSubtopic(number='3', text='\n    Find information about Bellevue Hospital Center in New York, NY.\n  ', type='inf'), TrecSubtopic(number='4', text='\n    Find the homepage of Bellevue University.\n  ', type='nav'), TrecSubtopic(number='5', text='\n    Find the homepage of Bellevue College, Washington.\n  ', type='nav'), TrecSubtopic(number='6', text='\n    Find the homepage of Bellevue Hospital Center in New York, NY.\n  ', type='nav'))),
            49: TrecWebTrackQuery('100', 'rincon puerto rico', '\n    Find information about Rincon, Puerto Rico.\n  ', 'faceted', (TrecSubtopic(number='1', text='\n    Find hotels and beach resorts in Rincon, Puerto Rico.\n  ', type='inf'), TrecSubtopic(number='2', text='\n    Find information on the history of Rincon, Puerto Rico.\n  ', type='inf'), TrecSubtopic(number='3', text='\n    Find surf forecasts for Rincon, Puerto Rico.\n  ', type='inf'), TrecSubtopic(number='4', text='\n    Find pictures of Rincon, Puerto Rico.\n  ', type='nav'), TrecSubtopic(number='5', text='\n    Find information about real estate and rental properties in\n    Rincon, Puerto Rico.\n  ', type='inf'))),
        })
        self._test_queries('clueweb09/en/trec-web-2011', count=50, items={
            0: TrecWebTrackQuery('101', 'ritz carlton lake las vegas', '\n    Find information about the Ritz Carlton resort at Lake Las Vegas.\n  ', 'faceted', (TrecSubtopic(number='1', text='\n    Find information about the Ritz Carlton resort at Lake Las Vegas.\n  ', type='inf'), TrecSubtopic(number='2', text='\n    Find a site where I can determine room price and availability.\n  ', type='nav'), TrecSubtopic(number='3', text='\n    Find directions to the Ritz Carlton Lake Las Vegas.\n  ', type='nav'), TrecSubtopic(number='4', text='\n    Find reviews of the Ritz Carlton Lake Las Vegas.\n  ', type='inf'))),
            9: TrecWebTrackQuery('110', 'map of brazil', '\n    What are the boundaries of the political jurisdictions in Brazil?\n  ', 'ambiguous', (TrecSubtopic(number='1', text='\n    What are the boundaries of the political jurisdictions in Brazil?\n  ', type='inf'), TrecSubtopic(number='2', text='\n    I am looking for information about taking a vacation trip to Brazil.\n  ', type='inf'), TrecSubtopic(number='3', text='\n    I want to buy a road map of Brazil.\n  ', type='nav'))),
            49: TrecWebTrackQuery('150', 'tn highway patrol', '\n    What are the requirements to become a Tennessee Highway Patrol State Trooper?\n  ', 'faceted', (TrecSubtopic(number='1', text='\n    What are the requirements to become a Tennessee Highway Patrol State Trooper?\n  ', type='inf'), TrecSubtopic(number='2', text='\n    information about the responsibilities of the Tennessee Highway Patrol\n  ', type='inf'), TrecSubtopic(number='3', text='\n    home page of the Tennessee Highway Patrol\n  ', type='nav'), TrecSubtopic(number='4', text='\n    I want to fill in the customer satisfaction survey about my interaction with a Tennessee Highway Patrol State Trooper.\n  ', type='nav'))),
        })
        self._test_queries('clueweb09/en/trec-web-2012', count=50, items={
            0: TrecWebTrackQuery('151', '403b', '\n    What is a 403b plan?\n  ', 'faceted', (TrecSubtopic(number='1', text='\n     What is a 403b plan?\n  ', type='inf'), TrecSubtopic(number='2', text='\n    Who is eligible for a 403b plan?\n  ', type='inf'), TrecSubtopic(number='3', text='\n    What are the rules for a 403b retirement plan?\n  ', type='nav'), TrecSubtopic(number='4', text='\n    What is the difference between 401k and 403b retirement plans?\n  ', type='inf'), TrecSubtopic(number='5', text='\n    What are the withdrawal limitations for a 403b retirement plan?\n  ', type='nav'))),
            9: TrecWebTrackQuery('160', 'grilling', '\n    Find kabob recipes.\n  ', 'ambiguous', (TrecSubtopic(number='1', text='\n    Find kabob recipes.\n  ', type='nav'), TrecSubtopic(number='2', text='\n    Find tips on grilling vegetables.\n  ', type='inf'), TrecSubtopic(number='3', text='\n    Find tips on grilling fish.\n  ', type='inf'), TrecSubtopic(number='4', text='\n    Find instructions for grilling chicken.\n  ', type='inf'), TrecSubtopic(number='5', text='\n    Find the Grilling Magazine website.\n  ', type='nav'), TrecSubtopic(number='6', text='\n    Find information on gas barbecue grills and cooking on a gas grill.\n  ', type='inf'))),
            49: TrecWebTrackQuery('200', 'ontario california airport', '\n    Find flight information for the Ontario, CA airport.\n  ', 'faceted', (TrecSubtopic(number='1', text='\n    Find flight information for the Ontario, CA airport.\n  ', type='inf'), TrecSubtopic(number='2', text='\n    What hotels are near the Ontario, CA airport?\n  ', type='inf'), TrecSubtopic(number='3', text='\n    What services/facilities does the Ontario, CA airport offer?\n  ', type='inf'), TrecSubtopic(number='4', text='\n    What is the address of the Ontario, CA airport?\n  ', type='nav'))),
        })
        self._test_queries('clueweb09/catb/trec-web-2009', count=50, items={
            0: TrecWebTrackQuery('1', 'obama family tree', "Find information on President Barack Obama's family\n  history, including genealogy, national origins, places and dates of\n  birth, etc.\n  ", 'faceted', (TrecSubtopic(number='1', text='\n    Find the TIME magazine photo essay "Barack Obama\'s Family Tree".\n  ', type='nav'), TrecSubtopic(number='2', text="\n    Where did Barack Obama's parents and grandparents come from?\n  ", type='inf'), TrecSubtopic(number='3', text="\n    Find biographical information on Barack Obama's mother.\n  ", type='inf'))),
            9: TrecWebTrackQuery('10', 'cheap internet', "I'm looking for cheap (i.e. low-cost) internet service.\n  ", 'faceted', (TrecSubtopic(number='1', text='\n    What are some low-cost broadband internet providers?\n  ', type='inf'), TrecSubtopic(number='2', text='\n    Do any internet providers still sell dial-up?\n  ', type='inf'), TrecSubtopic(number='3', text='\n    Who can provide inexpensive digital cable television bundled with\n    internet service?\n  ', type='inf'), TrecSubtopic(number='4', text="\n    I'm looking for the Vonage homepage.\n  ", type='nav'), TrecSubtopic(number='5', text='\n    Find me some providers of free wireless internet access.\n  ', type='inf'), TrecSubtopic(number='6', text='\n    I want to find cheap DSL providers.\n  ', type='inf'), TrecSubtopic(number='7', text='\n    Is there a way to get internet access without phone service?\n  ', type='inf'), TrecSubtopic(number='8', text="\n    Take me to Comcast's homepage.\n  ", type='nav'))),
            49: TrecWebTrackQuery('50', 'dog heat', 'What is the effect of excessive heat on dogs?\n  ', 'ambiguous', (TrecSubtopic(number='1', text='\n    What is the effect of excessive heat on dogs?\n  ', type='inf'), TrecSubtopic(number='2', text='\n    What are symptoms of heat stroke and other heat-related illnesses\n    in dogs?\n  ', type='inf'), TrecSubtopic(number='3', text='\n    Find information on dogs\' reproductive cycle.  What does it mean\n    when a dog is "in heat"?\n  ', type='inf'))),
        })
        self._test_queries('clueweb09/catb/trec-web-2010', count=50, items={
            0: TrecWebTrackQuery('51', 'horse hooves', '\n    Find information about horse hooves, their care, and diseases of hooves.\n  ', 'faceted', (TrecSubtopic(number='1', text="\n    Find information about horses' hooves and how to care for them.\n  ", type='inf'), TrecSubtopic(number='2', text='\n    Find pictures of horse hooves.\n  ', type='nav'), TrecSubtopic(number='3', text='\n    What are some injuries or diseases of hooves in horses, and how\n    are they treated?\n  ', type='inf'), TrecSubtopic(number='4', text="\n    Describe the anatomy of horses' feet and hooves.\n  ", type='inf'), TrecSubtopic(number='5', text='\n    Find information on shoeing horses and horseshoe problems.\n  ', type='inf'))),
            9: TrecWebTrackQuery('60', 'bellevue', '\n    Find information about Bellevue, Washington.\n  ', 'ambiguous', (TrecSubtopic(number='1', text='\n    Find information about Bellevue, Washington.\n  ', type='inf'), TrecSubtopic(number='2', text='\n    Find information about Bellevue, Nebraska.\n  ', type='inf'), TrecSubtopic(number='3', text='\n    Find information about Bellevue Hospital Center in New York, NY.\n  ', type='inf'), TrecSubtopic(number='4', text='\n    Find the homepage of Bellevue University.\n  ', type='nav'), TrecSubtopic(number='5', text='\n    Find the homepage of Bellevue College, Washington.\n  ', type='nav'), TrecSubtopic(number='6', text='\n    Find the homepage of Bellevue Hospital Center in New York, NY.\n  ', type='nav'))),
            49: TrecWebTrackQuery('100', 'rincon puerto rico', '\n    Find information about Rincon, Puerto Rico.\n  ', 'faceted', (TrecSubtopic(number='1', text='\n    Find hotels and beach resorts in Rincon, Puerto Rico.\n  ', type='inf'), TrecSubtopic(number='2', text='\n    Find information on the history of Rincon, Puerto Rico.\n  ', type='inf'), TrecSubtopic(number='3', text='\n    Find surf forecasts for Rincon, Puerto Rico.\n  ', type='inf'), TrecSubtopic(number='4', text='\n    Find pictures of Rincon, Puerto Rico.\n  ', type='nav'), TrecSubtopic(number='5', text='\n    Find information about real estate and rental properties in\n    Rincon, Puerto Rico.\n  ', type='inf'))),
        })
        self._test_queries('clueweb09/catb/trec-web-2011', count=50, items={
            0: TrecWebTrackQuery('101', 'ritz carlton lake las vegas', '\n    Find information about the Ritz Carlton resort at Lake Las Vegas.\n  ', 'faceted', (TrecSubtopic(number='1', text='\n    Find information about the Ritz Carlton resort at Lake Las Vegas.\n  ', type='inf'), TrecSubtopic(number='2', text='\n    Find a site where I can determine room price and availability.\n  ', type='nav'), TrecSubtopic(number='3', text='\n    Find directions to the Ritz Carlton Lake Las Vegas.\n  ', type='nav'), TrecSubtopic(number='4', text='\n    Find reviews of the Ritz Carlton Lake Las Vegas.\n  ', type='inf'))),
            9: TrecWebTrackQuery('110', 'map of brazil', '\n    What are the boundaries of the political jurisdictions in Brazil?\n  ', 'ambiguous', (TrecSubtopic(number='1', text='\n    What are the boundaries of the political jurisdictions in Brazil?\n  ', type='inf'), TrecSubtopic(number='2', text='\n    I am looking for information about taking a vacation trip to Brazil.\n  ', type='inf'), TrecSubtopic(number='3', text='\n    I want to buy a road map of Brazil.\n  ', type='nav'))),
            49: TrecWebTrackQuery('150', 'tn highway patrol', '\n    What are the requirements to become a Tennessee Highway Patrol State Trooper?\n  ', 'faceted', (TrecSubtopic(number='1', text='\n    What are the requirements to become a Tennessee Highway Patrol State Trooper?\n  ', type='inf'), TrecSubtopic(number='2', text='\n    information about the responsibilities of the Tennessee Highway Patrol\n  ', type='inf'), TrecSubtopic(number='3', text='\n    home page of the Tennessee Highway Patrol\n  ', type='nav'), TrecSubtopic(number='4', text='\n    I want to fill in the customer satisfaction survey about my interaction with a Tennessee Highway Patrol State Trooper.\n  ', type='nav'))),
        })
        self._test_queries('clueweb09/catb/trec-web-2012', count=50, items={
            0: TrecWebTrackQuery('151', '403b', '\n    What is a 403b plan?\n  ', 'faceted', (TrecSubtopic(number='1', text='\n     What is a 403b plan?\n  ', type='inf'), TrecSubtopic(number='2', text='\n    Who is eligible for a 403b plan?\n  ', type='inf'), TrecSubtopic(number='3', text='\n    What are the rules for a 403b retirement plan?\n  ', type='nav'), TrecSubtopic(number='4', text='\n    What is the difference between 401k and 403b retirement plans?\n  ', type='inf'), TrecSubtopic(number='5', text='\n    What are the withdrawal limitations for a 403b retirement plan?\n  ', type='nav'))),
            9: TrecWebTrackQuery('160', 'grilling', '\n    Find kabob recipes.\n  ', 'ambiguous', (TrecSubtopic(number='1', text='\n    Find kabob recipes.\n  ', type='nav'), TrecSubtopic(number='2', text='\n    Find tips on grilling vegetables.\n  ', type='inf'), TrecSubtopic(number='3', text='\n    Find tips on grilling fish.\n  ', type='inf'), TrecSubtopic(number='4', text='\n    Find instructions for grilling chicken.\n  ', type='inf'), TrecSubtopic(number='5', text='\n    Find the Grilling Magazine website.\n  ', type='nav'), TrecSubtopic(number='6', text='\n    Find information on gas barbecue grills and cooking on a gas grill.\n  ', type='inf'))),
            49: TrecWebTrackQuery('200', 'ontario california airport', '\n    Find flight information for the Ontario, CA airport.\n  ', 'faceted', (TrecSubtopic(number='1', text='\n    Find flight information for the Ontario, CA airport.\n  ', type='inf'), TrecSubtopic(number='2', text='\n    What hotels are near the Ontario, CA airport?\n  ', type='inf'), TrecSubtopic(number='3', text='\n    What services/facilities does the Ontario, CA airport offer?\n  ', type='inf'), TrecSubtopic(number='4', text='\n    What is the address of the Ontario, CA airport?\n  ', type='nav'))),
        })
        self._test_queries('clueweb09/trec-mq-2009', count=40000, items={
            0: GenericQuery('20001', '1:obama family tree'),
            9: GenericQuery('20010', '1:cheap internet'),
            39999: GenericQuery('60000', '4:bird shingles'),
        })

    def test_clueweb09_qrels(self):
        self._test_qrels('clueweb09/en/trec-web-2009', count=23601, items={
            0: TrecPrel('1', 'clueweb09-en0003-55-31884', 0, 0, 1.0),
            9: TrecPrel('1', 'clueweb09-en0009-84-37392', 0, 1, 0.0136322534877696),
            23600: TrecPrel('50', 'clueweb09-en0007-05-20194', 0, 1, 1.0),
        })
        self._test_qrels('clueweb09/en/trec-web-2010', count=25329, items={
            0: TrecQrel('51', 'clueweb09-en0000-16-19379', 0, '0'),
            9: TrecQrel('51', 'clueweb09-en0001-55-24197', 0, '0'),
            25328: TrecQrel('99', 'clueweb09-enwp03-23-18429', 0, '0'),
        })
        self._test_qrels('clueweb09/en/trec-web-2011', count=19381, items={
            0: TrecQrel('101', 'clueweb09-en0007-71-07471', 0, '0'),
            9: TrecQrel('101', 'clueweb09-en0044-05-29808', 2, '0'),
            19380: TrecQrel('150', 'clueweb09-en0003-86-25593', -2, '0'),
        })
        self._test_qrels('clueweb09/en/trec-web-2012', count=16055, items={
            0: TrecQrel('151', 'clueweb09-en0000-00-03430', -2, '0'),
            9: TrecQrel('151', 'clueweb09-en0000-00-04023', -2, '0'),
            16054: TrecQrel('200', 'clueweb09-enwp03-49-00268', 0, '0'),
        })
        self._test_qrels('clueweb09/catb/trec-web-2009', count=13118, items={
            0: TrecPrel('1', 'clueweb09-en0003-55-31884', 0, 0, 1.0),
            9: TrecPrel('1', 'clueweb09-enwp01-17-09993', 2, 1, 1.0),
            13117: TrecPrel('50', 'clueweb09-en0007-05-20194', 0, 1, 1.0),
        })
        self._test_qrels('clueweb09/catb/trec-web-2010', count=15845, items={
            0: TrecQrel('51', 'clueweb09-en0000-16-19379', 0, '0'),
            9: TrecQrel('51', 'clueweb09-en0001-55-24197', 0, '0'),
            15844: TrecQrel('99', 'clueweb09-enwp03-23-18429', 0, '0'),
        })
        self._test_qrels('clueweb09/catb/trec-web-2011', count=13081, items={
            0: TrecQrel('101', 'clueweb09-en0007-71-07471', 0, '0'),
            9: TrecQrel('101', 'clueweb09-en0001-12-16652', 0, '0'),
            13080: TrecQrel('150', 'clueweb09-en0003-86-25593', -2, '0'),
        })
        self._test_qrels('clueweb09/catb/trec-web-2012', count=10022, items={
            0: TrecQrel('151', 'clueweb09-en0000-00-03430', -2, '0'),
            9: TrecQrel('151', 'clueweb09-en0000-00-04023', -2, '0'),
            10021: TrecQrel('200', 'clueweb09-enwp03-49-00268', 0, '0'),
        })
        self._test_qrels('clueweb09/trec-mq-2009', count=34534, items={
            0: TrecPrel('20001', 'clueweb09-en0003-55-31884', 0, 0, 1.0),
            9: TrecPrel('20001', 'clueweb09-enwp01-17-09993', 2, 1, 1.0),
            34533: TrecPrel('57118', 'clueweb09-en0010-39-07801', 0, 1, 0.0612006151868131),
        })



if __name__ == '__main__':
    unittest.main()
