# -*- coding: utf-8 -*-
# These tests require a working internet connection.

from __future__ import unicode_literals
from __future__ import print_function
from __future__ import division

from builtins import str, bytes, dict, int
from builtins import map, zip, filter
from builtins import object, range, next

import os
import sys
sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
import unittest
import time
import warnings

from pattern import web

try:
    PATH = os.path.dirname(os.path.realpath(__file__))
except:
    PATH = ""

#---------------------------------------------------------------------------------------------------


class TestCache(unittest.TestCase):

    def setUp(self):
        pass

    def test_cache(self):
        # Assert cache unicode.
        k, v = "test", "ünîcødé"
        web.cache[k] = v
        self.assertTrue(isinstance(web.cache[k], str))
        self.assertEqual(web.cache[k], v)
        self.assertEqual(web.cache.age(k), 0)
        del web.cache[k]
        print("pattern.web.Cache")

#---------------------------------------------------------------------------------------------------


class TestUnicode(unittest.TestCase):

    def setUp(self):
        # Test data with different (or wrong) encodings.
        self.strings = (
            "ünîcøde",
            "ünîcøde".encode("utf-16"),
            "ünîcøde".encode("latin-1"),
            "ünîcøde".encode("windows-1252"),
             "ünîcøde",
            "אוניקאָד"
        )

    def test_decode_utf8(self):
        # Assert unicode.
        for s in self.strings:
            self.assertTrue(isinstance(web.decode_utf8(s), str))
        print("pattern.web.decode_utf8()")

    def test_encode_utf8(self):
        # Assert Python bytestring.
        for s in self.strings:
            self.assertTrue(isinstance(web.encode_utf8(s), bytes))
        print("pattern.web.encode_utf8()")

    def test_fix(self):
        # Assert fix for common Unicode mistakes.
        self.assertEqual(web.fix("cliché"), "cliché")
        self.assertEqual(web.fix("cliché"), "cliché")
        self.assertEqual(web.fix("cliché"), "cliché")
        self.assertEqual(web.fix("–"), "–")

#---------------------------------------------------------------------------------------------------


class TestURL(unittest.TestCase):

    def setUp(self):
        # Test a live URL that has fast response time
        self.live = "http://www.google.com/"
        # Test a fake URL with the URL parser.
        self.url = "https://username:password@www.domain.com:8080/path/path/page.html?q=1#anchor"
        self.parts = {
            "protocol": "https",
            "username": "username",
            "password": "password",
              "domain": "www.domain.com",
                "port": 8080,
                "path": ["path", "path"],
                "page": "page.html",
               "query": {"q": 1},
              "anchor": "anchor"
        }

    def test_asynchrous(self):
        # Assert asynchronous function call (returns 1).
        v = web.asynchronous(lambda t: time.sleep(t) or 1, 0.2)
        while not v.done:
            time.sleep(0.1)
        self.assertEqual(v.value, 1)
        print("pattern.web.asynchronous()")

    def test_extension(self):
        # Assert filename extension.
        v = web.extension(os.path.join("pattern", "test", "test-web.py.zip"))
        self.assertEqual(v, ".zip")
        print("pattern.web.extension()")

    def test_urldecode(self):
        # Assert URL decode (inverse of urllib.urlencode).
        v = web.urldecode("?user=me&page=1&q=&")
        self.assertEqual(v, {"user": "me", "page": 1, "q": None})
        print("pattern.web.urldecode()")

    def test_proxy(self):
        # Assert URL proxy.
        v = web.proxy("www.proxy.com", "https")
        self.assertEqual(v, ("www.proxy.com", "https"))
        print("pattern.web.proxy()")

    def test_url_parts(self):
        # Assert URL._parse and URL.parts{}.
        v = web.URL(self.url)
        for a, b in (
          (web.PROTOCOL, self.parts["protocol"]),
          (web.USERNAME, self.parts["username"]),
          (web.PASSWORD, self.parts["password"]),
          (web.DOMAIN,   self.parts["domain"]),
          (web.PORT,     self.parts["port"]),
          (web.PATH,     self.parts["path"]),
          (web.PAGE,     self.parts["page"]),
          (web.QUERY,    self.parts["query"]),
          (web.ANCHOR,   self.parts["anchor"])):
            self.assertEqual(v.parts[a], b)
        print("pattern.web.URL.parts")

    def test_url_query(self):
        # Assert URL.query and URL.querystring.
        v = web.URL(self.url)
        v.query["page"] = 10
        v.query["user"] = None
        self.assertEqual(v.query, {"q": 1, "page": 10, "user": None})
        self.assertEqual(v.querystring, "q=1&page=10&user=")
        # Assert URL.querystring encodes unicode arguments.
        q = ({"ünîcødé": 1.5}, "%C3%BCn%C3%AEc%C3%B8d%C3%A9=1.5")
        v.query = q[0]
        self.assertEqual(v.querystring, q[1])
        # Assert URL.query decodes unicode arguments.
        v = web.URL("http://domain.com?" + q[1])
        self.assertEqual(v.query, q[0])
        print("pattern.web.URL.query")
        print("pattern.web.URL.querystring")

    def test_url_string(self):
        # Assert URL._set_string().
        v = web.URL("")
        v.string = "https://domain.com"
        self.assertEqual(v.parts[web.PROTOCOL], "https")
        self.assertEqual(v.parts[web.DOMAIN], "domain.com")
        self.assertEqual(v.parts[web.PATH], [])
        print("pattern.web.URL.string")

    def test_url(self):
        # Assert URL.copy().
        v = web.URL(self.url)
        v = v.copy()
        # Assert URL.__setattr__().
        v.username = "new-username"
        v.password = "new-password"
        # Assert URL.__getattr__().
        self.assertEqual(v.method, web.GET)
        self.assertEqual(v.protocol, self.parts["protocol"])
        self.assertEqual(v.username, "new-username")
        self.assertEqual(v.password, "new-password")
        self.assertEqual(v.domain, self.parts["domain"])
        self.assertEqual(v.port, self.parts["port"])
        self.assertEqual(v.path, self.parts["path"])
        self.assertEqual(v.page, self.parts["page"])
        self.assertEqual(v.query, self.parts["query"])
        self.assertEqual(v.anchor, self.parts["anchor"])
        print("pattern.web.URL")

    def test_url_open(self):
        # Assert URLError.
        v = web.URL(self.live.replace("http://", "htp://"))
        self.assertRaises(web.URLError, v.open)
        self.assertEqual(v.exists, False)
        # Assert HTTPError.
        v = web.URL(self.live + "iphone/android.html")
        self.assertRaises(web.HTTPError, v.open)
        self.assertRaises(web.HTTP404NotFound, v.open)
        self.assertEqual(v.exists, False)
        # Assert socket connection.
        v = web.URL(self.live)
        self.assertTrue(v.open() is not None)
        self.assertEqual(v.exists, True)
        # Assert user-agent and referer.
        self.assertTrue(v.open(user_agent=web.MOZILLA, referrer=web.REFERRER) is not None)
        print("pattern.web.URL.exists")
        print("pattern.web.URL.open()")

    def test_url_download(self):
        t = time.time()
        v = web.URL(self.live).download(cached=False, throttle=0.25, unicode=True)
        t = time.time() - t
        # Assert unicode content.
        self.assertTrue(isinstance(v, str))
        # Assert download rate limiting.
        self.assertTrue(t >= 0.25)
        print("pattern.web.URL.download()")

    def test_url_mimetype(self):
        # Assert URL MIME-type.
        v = web.URL(self.live).mimetype
        self.assertTrue(v in web.MIMETYPE_WEBPAGE)
        print("pattern.web.URL.mimetype")

    def test_url_headers(self):
        # Assert URL headers.
        v = web.URL(self.live).headers["content-type"].split(";")[0]
        self.assertEqual(v, "text/html")
        print("pattern.web.URL.headers")

    def test_url_redirect(self):
        # Assert URL redirected URL (this depends on where you are).
        # In Belgium, it yields "http://www.google.be/".
        v = web.URL(self.live).redirect
        print("pattern.web.URL.redirect: " + self.live + " => " + str(v))

    def test_abs(self):
        # Assert absolute URL (special attention for anchors).
        for a, b in (
          ("../page.html", "http://domain.com/path/"),
          (   "page.html", "http://domain.com/home.html")):
            v = web.abs(a, base=b)
            self.assertEqual(v, "http://domain.com/page.html")
        for a, b, c in (
          (     "#anchor", "http://domain.com", "/"),
          (     "#anchor", "http://domain.com/", ""),
          (     "#anchor", "http://domain.com/page", "")):
            v = web.abs(a, base=b)
            self.assertEqual(v, b + c + a) # http://domain.com/#anchor
        print("pattern.web.abs()")

    def test_base(self):
        # Assert base URL domain name.
        self.assertEqual(web.base("http://domain.com/home.html"), "domain.com")
        print("pattern.web.base()")

    def test_oauth(self):
        # Assert OAuth algorithm.
        data = {
            "q": '"cåts, døgs & chîckéns = fün+"',
            "oauth_version": "1.0",
            "oauth_nonce": "0",
            "oauth_timestamp": 0,
            "oauth_consumer_key": "key",
            "oauth_signature_method": "HMAC-SHA1"
        }
        v = web.oauth.sign("http://yboss.yahooapis.com/ysearch/web", data, secret="secret")
        self.assertEqual(v, "RtTu8dxSp3uBzSbsuLAXIWOKfyI=")
        print("pattern.web.oauth.sign()")

#---------------------------------------------------------------------------------------------------


class TestPlaintext(unittest.TestCase):

    def setUp(self):
        pass

    def test_find_urls(self):
        # Assert URL finder with common URL notations.
        for url in (
          "http://domain.co.uk",
          "https://domain.co.uk",
          "www.domain.cu.uk",
          "domain.com",
          "domain.org",
          "domain.net"):
            self.assertEqual(web.find_urls("(" + url + ".")[0], url)
        # Assert case-insensitive, punctuation and <a href="">.
        # Assert several matches in string.
        self.assertEqual(web.find_urls("HTTP://domain.net")[0], "HTTP://domain.net")
        self.assertEqual(web.find_urls("http://domain.net),};")[0], "http://domain.net")
        self.assertEqual(web.find_urls("http://domain.net\">domain")[0], "http://domain.net")
        self.assertEqual(web.find_urls("domain.com, domain.net"), ["domain.com", "domain.net"])
        print("pattern.web.find_urls()")

    def test_find_email(self):
        # Assert e-mail finder with common e-mail notations.
        s = "firstname.last+name@domain.ac.co.uk"
        v = web.find_email("(" + s + ".")
        self.assertEqual(v[0], s)
        # Assert several matches in string.
        s = ["me@site1.com", "me@site2.com"]
        v = web.find_email("(" + ",".join(s) + ")")
        self.assertEqual(v, s)
        print("pattern.web.find_email()")

    def test_find_between(self):
        # Assert search between open tag and close tag.
        s = "<script type='text/javascript'>alert(0);</script>"
        v = web.find_between("<script", "</script>", s)
        self.assertEqual(v[0], " type='text/javascript'>alert(0);")
        # Assert several matches in string.
        s = "a0ba1b"
        v = web.find_between("a", "b", s)
        self.assertEqual(v, ["0", "1"])
        print("pattern.web.find_between()")

    def test_strip_tags(self):
        # Assert HTML parser and tag stripper.
        for html, plain in (
          ("<b>ünîcøde</b>", "ünîcøde"),
          ("<img src=""/>", ""),
          ("<p>text</p>", "text\n\n"),
          ("<li>text</li>", "* text\n"),
          ("<td>text</td>", "text\t"),
          ("<br>", "\n"),
          ("<br/>", "\n\n"),
          ("<br /><br/><br>", "\n\n\n\n\n")):
            self.assertEqual(web.strip_tags(html), plain)
        # Assert exclude tags and attributes
        v = web.strip_tags("<a href=\"\" onclick=\"\">text</a>", exclude={"a": ["href"]})
        self.assertEqual(v, "<a href=\"\">text</a>")
        print("pattern.web.strip_tags()")

    def test_strip_element(self):
        # Assert strip <p> elements.
        v = web.strip_element(" <p><p></p>text</p> <b><P></P></b>", "p")
        self.assertEqual(v, "  <b></b>")
        print("pattern.web.strip_element()")

    def test_strip_between(self):
        # Assert strip <p> elements.
        v = web.strip_between("<p", "</p>", " <p><p></p>text</p> <b><P></P></b>")
        self.assertEqual(v, " text</p> <b></b>")
        print("pattern.web.strip_between()")

    def test_strip_javascript(self):
        # Assert strip <script> elements.
        v = web.strip_javascript(" <script type=\"text/javascript\">text</script> ")
        self.assertEqual(v, "  ")
        print("pattern.web.strip_javascript()")

    def test_strip_inline_css(self):
        # Assert strip <style> elements.
        v = web.strip_inline_css(" <style type=\"text/css\">text</style> ")
        self.assertEqual(v, "  ")
        print("pattern.web.strip_inline_css()")

    def test_strip_comments(self):
        # Assert strip <!-- --> elements.
        v = web.strip_comments(" <!-- text --> ")
        self.assertEqual(v, "  ")
        print("pattern.web.strip_comments()")

    def test_strip_forms(self):
        # Assert strip <form> elements.
        v = web.strip_forms(" <form method=\"get\">text</form> ")
        self.assertEqual(v, "  ")
        print("pattern.web.strip_forms()")

    def test_encode_entities(self):
        # Assert HTML entity encoder (e.g., "&" => "&&amp;")
        for a, b in (
          ("&#201;", "&#201;"),
          ("&", "&amp;"),
          ("<", "&lt;"),
          (">", "&gt;"),
          ('"', "&quot;"),
          ("'", "&#39;")):
            self.assertEqual(web.encode_entities(a), b)
        print("pattern.web.encode_entities()")

    def test_decode_entities(self):
        # Assert HMTL entity decoder (e.g., "&amp;" => "&")
        for a, b in (
          ("&#38;", "&"),
          ("&amp;", "&"),
          ("&#x0026;", "&"),
          ("&#160;", "\xa0"),
          ("&foo;", "&foo;")):
            self.assertEqual(web.decode_entities(a), b)
        print("pattern.web.decode_entities()")

    def test_collapse_spaces(self):
        # Assert collapse multiple spaces.
        for a, b in (
          ("    ", ""),
          (" .. ", ".."),
          (".  .", ". ."),
          (". \n", "."),
          ("\xa0", "")):
            self.assertEqual(web.collapse_spaces(a), b)
        # Assert preserve indendation.
        self.assertEqual(web.collapse_spaces("  . \n", indentation=True), "  .")
        print("pattern.web.collapse_spaces()")

    def test_collapse_tabs(self):
        # Assert collapse multiple tabs to 1 space.
        for a, b in (
          ("\t\t\t", ""),
          ("\t..\t", ".."),
          (".\t\t.", ". ."),
          (".\t\n", ".")):
            self.assertEqual(web.collapse_tabs(a), b)
        # Assert preserve indendation.
        self.assertEqual(web.collapse_tabs("\t\t .\t\n", indentation=True), "\t\t .")
        print("pattern.web.collapse_tabs()")

    def test_collapse_linebreaks(self):
        # Assert collapse multiple linebreaks.
        for a, b in (
          ("\n\n\n", "\n"),
          (".\n\n.", ".\n."),
          (".\r\n.", ".\n."),
          (".\n  .", ".\n  ."),
          (" \n  .", "\n  .")):
            self.assertEqual(web.collapse_linebreaks(a), b)
        print("pattern.web.collapse_linebreaks()")

    def test_plaintext(self):
        # Assert plaintext:
        # - strip <script>, <style>, <form>, <!-- --> elements,
        # - strip tags,
        # - decode entities,
        # - collapse whitespace,
        html = """
            <html>
            <head>
                <title>tags &amp; things</title>
            </head>
            <body>
                <div id="content">       \n\n\n\
                    <!-- main content -->
                    <script type="text/javascript>"alert(0);</script>
                    <h1>title1</h1>
                    <h2>title2</h2>
                    <p>paragraph1</p>
                    <p>paragraph2 <a href="http://www.domain.com" onclick="alert(0);">link</a></p>
                    <ul>
                        <li>item1&nbsp;&nbsp;&nbsp;xxx</li>
                        <li>item2</li>
                    <ul>
                </div>
                <br />
                <br />
            </body>
            </html>
        """
        self.assertEqual(web.plaintext(html, keep={"a": "href"}),
            "tags & things\n\ntitle1\n\ntitle2\n\nparagraph1\n\nparagraph2 " + \
            "<a href=\"http://www.domain.com\">link</a>\n\n* item1 xxx\n* item2")
        print("pattern.web.plaintext()")

#---------------------------------------------------------------------------------------------------


class TestSearchEngine(unittest.TestCase):

    def setUp(self):
        # Test data for all search engines:
        # {api: (source, license, Engine)}.
        self.api = {
            "Google": (web.GOOGLE,      web.GOOGLE_LICENSE,      web.Google),
             "Yahoo": (web.YAHOO,       web.YAHOO_LICENSE,       web.Yahoo),
              "Bing": (web.BING,        web.BING_LICENSE,        web.Bing),
           "Twitter": (web.TWITTER,     web.TWITTER_LICENSE,     web.Twitter),
         "Wikipedia": (web.MEDIAWIKI,   web.WIKIPEDIA_LICENSE,   web.Wikipedia),
             "Wikia": (web.MEDIAWIKI,   web.MEDIAWIKI_LICENSE,   web.Wikia),
            "Flickr": (web.FLICKR,      web.FLICKR_LICENSE,      web.Flickr),
          "Facebook": (web.FACEBOOK,    web.FACEBOOK_LICENSE,    web.Facebook),
       "ProductWiki": (web.PRODUCTWIKI, web.PRODUCTWIKI_LICENSE, web.ProductWiki)
        }

    def _test_search_engine(self, api, source, license, Engine, query="today", type=web.SEARCH):
        # Assert SearchEngine standard interface for any api:
        # Google, Yahoo, Bing, Twitter, Wikipedia, Flickr, Facebook, ProductWiki, Newsfeed.
        # SearchEngine.search() returns a list of Result objects with unicode fields,
        # except Wikipedia which returns a WikipediaArticle (MediaWikiArticle subclass).
        if api == "Yahoo" and license == ("", ""):
            return
        t = time.time()
        e = Engine(license=license, throttle=0.25, language="en")
        v = e.search(query, type, start=1, count=1, cached=False)
        t = time.time() - t
        self.assertTrue(t >= 0.25)
        self.assertEqual(e.license, license)
        self.assertEqual(e.throttle, 0.25)
        self.assertEqual(e.language, "en")
        self.assertEqual(v.query, query)
        if source != web.MEDIAWIKI:
            self.assertEqual(v.source, source)
            self.assertEqual(v.type, type)
            self.assertEqual(len(v), 1)
            self.assertTrue(isinstance(v[0], web.Result))
            self.assertTrue(isinstance(v[0].url, str))
            self.assertTrue(isinstance(v[0].title, str))
            self.assertTrue(isinstance(v[0].description, str))
            self.assertTrue(isinstance(v[0].language, str))
            self.assertTrue(isinstance(v[0].author, (str, tuple)))
            self.assertTrue(isinstance(v[0].date, str))
        else:
            self.assertTrue(isinstance(v, web.MediaWikiArticle))
        # Assert zero results for start < 1 and count < 1.
        v1 = e.search(query, start=0)
        v2 = e.search(query, count=0)
        if source != web.MEDIAWIKI:
            self.assertEqual(len(v1), 0)
            self.assertEqual(len(v2), 0)
        else:
            self.assertTrue(isinstance(v1, web.MediaWikiArticle))
            self.assertEqual(v2, None)
        # Assert SearchEngineTypeError for unknown type.
        self.assertRaises(web.SearchEngineTypeError, e.search, query, type="crystall-ball")
        print("pattern.web.%s.search()" % api)

    def test_search_google(self):
        self._test_search_engine("Google", *self.api["Google"])

    def test_search_yahoo(self):
        self._test_search_engine("Yahoo", *self.api["Yahoo"])

    @unittest.skip('Bing Search API has no free quota')
    def test_search_bing(self):
        self._test_search_engine("Bing", *self.api["Bing"])

    def test_search_twitter(self):
        self._test_search_engine("Twitter", *self.api["Twitter"])

    @unittest.skip('Mediawiki/Wikipedia API or appearance changed')
    def test_search_wikipedia(self):
        self._test_search_engine("Wikipedia", *self.api["Wikipedia"])

    @unittest.skip('Mediawiki API or appearance changed')
    def test_search_wikia(self):
        self._test_search_engine("Wikia", *self.api["Wikia"], **{"query": "games"})

    def test_search_flickr(self):
        self._test_search_engine("Flickr", *self.api["Flickr"], **{"type": web.IMAGE})

    @unittest.skip('Facebook API changed')
    def test_search_facebook(self):
        self._test_search_engine("Facebook", *self.api["Facebook"])

    @unittest.skip('ProductWiki is deprecated')
    def test_search_productwiki(self):
        self._test_search_engine("ProductWiki", *self.api["ProductWiki"], **{"query": "computer"})

    def test_search_newsfeed(self):
        for feed, url in web.feeds.items():
            self._test_search_engine("Newsfeed", url, None, web.Newsfeed, query=url, type=web.NEWS)

    def _test_results(self, api, source, license, Engine, type=web.SEARCH, query="today", baseline=[6, 6, 6, 0]):
        # Assert SearchEngine result content.
        # We expect to find http:// URL's and descriptions containing the search query.
        if api == "Yahoo" and license == ("", ""):
            return
        i1 = 0
        i2 = 0
        i3 = 0
        i4 = 0
        e = Engine(license=license, language="en", throttle=0.25)
        for result in e.search(query, type, count=10, cached=False):
            i1 += int(result.url.startswith("http"))
            i2 += int(query in result.url.lower())
            i2 += int(query in result.title.lower())
            i2 += int(query in result.description.lower())
            i3 += int(result.language == "en")
            i4 += int(result.url.endswith(("jpg", "png", "gif")))
            #print(result.url)
            #print(result.title)
            #print(result.description)
        #print(i1, i2, i3, i4)
        self.assertTrue(i1 >= baseline[0]) # url's starting with "http"
        self.assertTrue(i2 >= baseline[1]) # query in url + title + description
        self.assertTrue(i3 >= baseline[2]) # language "en"
        self.assertTrue(i4 >= baseline[3]) # url's ending with "jpg", "png" or "gif"
        print("pattern.web.%s.Result(type=%s)" % (api, type.upper()))

    def test_results_google(self):
        self._test_results("Google", *self.api["Google"])

    def test_results_yahoo(self):
        self._test_results("Yahoo", *self.api["Yahoo"])

    def test_results_yahoo_images(self):
        self._test_results("Yahoo", *self.api["Yahoo"], **{"type": web.IMAGE, "baseline": [6, 6, 0, 6]})

    def test_results_yahoo_news(self):
        self._test_results("Yahoo", *self.api["Yahoo"], **{"type": web.NEWS})

    @unittest.skip('Bing API changed')
    def test_results_bing(self):
        self._test_results("Bing", *self.api["Bing"])

    @unittest.skip('Bing API changed')
    def test_results_bing_images(self):
        self._test_results("Bing", *self.api["Bing"], **{"type": web.IMAGE, "baseline": [6, 6, 0, 6]})

    @unittest.skip('Bing API changed')
    def test_results_bing_news(self):
        self._test_results("Bing", *self.api["Bing"], **{"type": web.NEWS})

    def test_results_twitter(self):
        self._test_results("Twitter", *self.api["Twitter"])

    def test_results_flickr(self):
        self._test_results("Flickr", *self.api["Flickr"], **{"baseline": [6, 6, 0, 6]})

    @unittest.skip('Facebook API changed')
    def test_results_facebook(self):
        self._test_results("Facebook", *self.api["Facebook"], **{"baseline": [0, 1, 0, 0]})

    def test_google_translate(self):
        try:
            # Assert Google Translate API.
            # Requires license with billing enabled.
            source, license, Engine = self.api["Google"]
            v = Engine(license, throttle=0.25).translate("thé", input="fr", output="en", cached=False)
            self.assertEqual(v, "tea")
            print("pattern.web.Google.translate()")
        except web.HTTP401Authentication:
            pass

    def test_google_identify(self):
        try:
            # Assert Google Translate API (language detection).
            # Requires license with billing enabled.
            source, license, Engine = self.api["Google"]
            v = Engine(license, throttle=0.25).identify("L'essence des mathématiques, c'est la liberté!", cached=False)
            self.assertEqual(v[0], "fr")
            print("pattern.web.Google.identify()")
        except web.HTTP401Authentication:
            pass

    def test_twitter_author(self):
        self.assertEqual(web.author("me"), "from:me")
        print("pattern.web.author()")

    def test_twitter_hashtags(self):
        self.assertEqual(web.hashtags("#cat #dog"), ["#cat", "#dog"])
        print("pattern.web.hashtags()")

    def test_twitter_retweets(self):
        self.assertEqual(web.retweets("RT @me: blah"), ["@me"])
        print("pattern.web.retweets()")

    def _test_search_image_size(self, api, source, license, Engine):
        # Assert image URL's for different sizes actually exist.
        if api == "Yahoo" and license == ("", ""):
            return
        e = Engine(license, throttle=0.25)
        for size in (web.TINY, web.SMALL, web.MEDIUM, web.LARGE):
            v = e.search("cats", type=web.IMAGE, count=1, size=size, cached=False)
            self.assertEqual(web.URL(v[0].url).exists, True)
            print("pattern.web.%s.search(type=IMAGE, size=%s)" % (api, size.upper()))

    def test_yahoo_image_size(self):
        self._test_search_image_size("Yahoo", *self.api["Yahoo"])

    @unittest.skip('Bing Search API has no free quota')
    def test_bing_image_size(self):
        self._test_search_image_size("Bing", *self.api["Bing"])

    def test_flickr_image_size(self):
        self._test_search_image_size("Flickr", *self.api["Flickr"])

    @unittest.skip('Mediawiki/Wikipedia API or appearance changed')
    def test_wikipedia_list(self):
        # Assert WikipediaArticle.list(), an iterator over all article titles.
        source, license, Engine = self.api["Wikipedia"]
        v = Engine(license).list(start="a", count=1)
        v = [next(v) for i in range(2)]
        self.assertTrue(len(v) == 2)
        self.assertTrue(v[0].lower().startswith("a"))
        self.assertTrue(v[1].lower().startswith("a"))
        print("pattern.web.Wikipedia.list()")

    def test_wikipedia_all(self):
        # Assert WikipediaArticle.all(), an iterator over WikipediaArticle objects.
        source, license, Engine = self.api["Wikipedia"]
        v = Engine(license).all(start="a", count=1)
        v = [next(v) for i in range(1)]
        self.assertTrue(len(v) == 1)
        self.assertTrue(isinstance(v[0], web.WikipediaArticle))
        self.assertTrue(v[0].title.lower().startswith("a"))
        print("pattern.web.Wikipedia.all()")

    @unittest.skip('Mediawiki/Wikipedia API or appearance changed')
    def test_wikipedia_article(self):
        source, license, Engine = self.api["Wikipedia"]
        v = Engine(license).search("cat", cached=False)
        # Assert WikipediaArticle properties.
        self.assertTrue(isinstance(v.title, str))
        self.assertTrue(isinstance(v.string, str))
        self.assertTrue(isinstance(v.links, list))
        self.assertTrue(isinstance(v.categories, list))
        self.assertTrue(isinstance(v.external, list))
        self.assertTrue(isinstance(v.media, list))
        self.assertTrue(isinstance(v.languages, dict))
        # Assert WikipediaArticle properties content.
        self.assertTrue(v.string == v.plaintext())
        self.assertTrue(v.html == v.source)
        self.assertTrue("</div>"  in v.source)
        self.assertTrue("cat"     in v.title.lower())
        self.assertTrue("Felis"   in v.links)
        self.assertTrue("Felines" in v.categories)
        self.assertTrue("en" == v.language)
        self.assertTrue("fr"      in v.languages)
        self.assertTrue("chat"    in v.languages["fr"].lower())
        self.assertTrue(v.external[0].startswith("http"))
        self.assertTrue(v.media[0].endswith(("jpg", "png", "gif", "svg")))
        print("pattern.web.WikipediaArticle")

    @unittest.skip('Mediawiki/Wikipedia API or appearance changed')
    def test_wikipedia_article_sections(self):
        # Assert WikipediaArticle.sections structure.
        # The test may need to be modified if the Wikipedia "Cat" article changes.
        source, license, Engine = self.api["Wikipedia"]
        v = Engine(license).search("cat", cached=False)
        s1 = s2 = s3 = None
        for section in v.sections:
            if section.title == "Behavior":
                s1 = section
            if section.title == "Grooming":
                s2 = section
            if section.title == "Play":
                s3 = section
            self.assertTrue(section.article == v)
            self.assertTrue(section.level == 0 or section.string.startswith(section.title))
        # Test section depth.
        self.assertTrue(s1.level == 1)
        self.assertTrue(s2.level == 2)
        self.assertTrue(s2.level == 2)
        # Test section parent-child structure.
        self.assertTrue(s2 in s1.children) # Behavior => Grooming
        self.assertTrue(s3 in s1.children) # Behavior => Play
        self.assertTrue(s2.parent == s1)
        self.assertTrue(s3.parent == s1)
        # Test section content.
        self.assertTrue("hairballs" in s2.content)
        self.assertTrue("laser pointer" in s3.content)
        # Test section tables.
        # XXX should test <td colspan="x"> more thoroughly.
        self.assertTrue(len(v.sections[1].tables) > 0)
        print("pattern.web.WikipediaSection")

    @unittest.skip('ProductWiki is deprecated')
    def test_productwiki(self):
        # Assert product reviews and score.
        source, license, Engine = self.api["ProductWiki"]
        v = Engine(license).search("computer", cached=False)
        self.assertTrue(isinstance(v[0].reviews, list))
        self.assertTrue(isinstance(v[0].score, int))
        print("pattern.web.ProductWiki.Result.reviews")
        print("pattern.web.ProductWiki.Result.score")

#---------------------------------------------------------------------------------------------------


class TestDOM(unittest.TestCase):

    def setUp(self):
        # Test HTML document.
        self.html = """
            <!doctype html>
            <html lang="en">
            <head>
                <title>title</title>
                <meta charset="utf-8" />
            </head>
            <body id="front" class="comments">
                <script type="text/javascript">alert(0);</script>
                <div id="navigation">
                    <a href="nav1.html">nav1</a> | 
                    <a href="nav2.html">nav2</a> | 
                    <a href="nav3.html">nav3</a>
                </div>
                <div id="content">
                    <P class="comment">
                        <span class="date">today</span>
                        <span class="author">me</span>
                        Blah blah
                    </P>
                    <P class="class1 class2">
                        Blah blah
                    </P>
                    <p>Read more</p>
                </div>
            </body>
            </html>
        """

    def test_node_document(self):
        # Assert Node properties.
        v1 = web.Document(self.html)
        self.assertEqual(v1.type, web.DOCUMENT)
        self.assertEqual(v1.source[:10], "<!DOCTYPE ") # Note: BeautifulSoup strips whitespace.
        self.assertEqual(v1.parent, None)
        # Assert Node traversal.
        v2 = v1.children[0].next
        self.assertEqual(v2.type, web.ELEMENT)
        self.assertEqual(v2.previous, v1.children[0])
        # Assert Document properties.
        v3 = v1.declaration
        self.assertEqual(v3, v1.children[0])
        self.assertEqual(v3.parent, v1)
        self.assertEqual(v3.source, "html")
        self.assertEqual(v1.head.type, web.ELEMENT)
        self.assertEqual(v1.body.type, web.ELEMENT)
        self.assertTrue(v1.head.source.startswith("<head"))
        self.assertTrue(v1.body.source.startswith("<body"))
        print("pattern.web.Node")
        print("pattern.web.DOM")

    def test_node_traverse(self):
        # Assert Node.traverse() (must visit all child nodes recursively).
        self.b = False

        def visit(node):
            if node.type == web.ELEMENT and node.tag == "span":
                self.b = True
        v = web.DOM(self.html)
        v.traverse(visit)
        self.assertEqual(self.b, True)
        print("pattern.web.Node.traverse()")

    def test_element(self):
        # Assert Element properties (test <body>).
        v = web.DOM(self.html).body
        self.assertEqual(v.tag, "body")
        self.assertEqual(v.attributes["id"], "front")
        self.assertEqual(v.attributes["class"], ["comments"])
        self.assertTrue(v.content.startswith("\n<script"))
        # Assert Element.getElementsByTagname() (test navigation links).
        a = v.by_tag("a")
        self.assertEqual(len(a), 3)
        self.assertEqual(a[0].content, "nav1")
        self.assertEqual(a[1].content, "nav2")
        self.assertEqual(a[2].content, "nav3")
        # Assert Element.getElementsByClassname() (test <p class="comment">).
        a = v.by_class("comment")
        self.assertEqual(a[0].tag, "p")
        self.assertEqual(a[0].by_tag("span")[0].attributes["class"], ["date"])
        self.assertEqual(a[0].by_tag("span")[1].attributes["class"], ["author"])
        for selector in (".comment", "p.comment", "*.comment"):
            self.assertEqual(v.by_tag(selector)[0], a[0])
        # Assert Element.getElementById() (test <div id="content">).
        e = v.by_id("content")
        self.assertEqual(e.tag, "div")
        self.assertEqual(e, a[0].parent)
        for selector in ("#content", "div#content", "*#content"):
            self.assertEqual(v.by_tag(selector)[0], e)
        # Assert Element.getElementByAttribute() (test on <a href="">).
        a = v.by_attribute(href="nav1.html")
        self.assertEqual(a[0].content, "nav1")
        print("pattern.web.Element")
        print("pattern.web.Element.by_tag()")
        print("pattern.web.Element.by_class()")
        print("pattern.web.Element.by_id()")
        print("pattern.web.Element.by_attribute()")

    def test_selector(self):
        # Assert DOM CSS selectors with multiple classes.
        v = web.DOM(self.html).body
        p = v("p.class1")
        self.assertEqual(len(p), 1)
        self.assertTrue("class1" in p[0].attributes["class"])
        p = v("p.class2")
        self.assertEqual(len(p), 1)
        self.assertTrue("class2" in p[0].attributes["class"])
        p = v("p.class1.class2")
        self.assertEqual(len(p), 1)
        self.assertTrue("class1" in p[0].attributes["class"])
        self.assertTrue("class2" in p[0].attributes["class"])
        e = p[0]
        self.assertEqual(e, v("p[class='class1 class2']")[0])
        self.assertEqual(e, v("p[class^='class1']")[0])
        self.assertEqual(e, v("p[class$='class2']")[0])
        self.assertEqual(e, v("p[class*='class']")[0])
        self.assertEqual(e, v("p:contains('blah')")[1])
        self.assertTrue(web.Selector("p[class='class1 class2']").match(e))
        print("pattern.web.Selector()")

#---------------------------------------------------------------------------------------------------


class TestDocumentParser(unittest.TestCase):

    def setUp(self):
        pass

    def test_pdf(self):
        # Assert PDF to string parser.
        s = web.parsedoc(os.path.join(PATH, "corpora", "carroll-wonderland.pdf"))
        self.assertTrue("Curiouser and curiouser!" in s)
        self.assertTrue(isinstance(s, str))
        print("pattern.web.parsepdf()")

    def test_docx(self):
        # Assert PDF to string parser.
        s = web.parsedoc(os.path.join(PATH, "corpora", "carroll-lookingglass.docx"))
        self.assertTrue("'Twas brillig, and the slithy toves" in s)
        self.assertTrue(isinstance(s, str))
        print("pattern.web.parsedocx()")

#---------------------------------------------------------------------------------------------------


class TestLocale(unittest.TestCase):

    def setUp(self):
        pass

    def test_encode_language(self):
        # Assert "Dutch" => "nl".
        self.assertEqual(web.locale.encode_language("dutch"), "nl")
        self.assertEqual(web.locale.encode_language("?????"), None)
        print("pattern.web.locale.encode_language()")

    def test_decode_language(self):
        # Assert "nl" => "Dutch".
        self.assertEqual(web.locale.decode_language("nl"), "Dutch")
        self.assertEqual(web.locale.decode_language("NL"), "Dutch")
        self.assertEqual(web.locale.decode_language("??"), None)
        print("pattern.web.locale.decode_language()")

    def test_encode_region(self):
        # Assert "Belgium" => "BE".
        self.assertEqual(web.locale.encode_region("belgium"), "BE")
        self.assertEqual(web.locale.encode_region("???????"), None)
        print("pattern.web.locale.encode_region()")

    def test_decode_region(self):
        # Assert "BE" => "Belgium".
        self.assertEqual(web.locale.decode_region("be"), "Belgium")
        self.assertEqual(web.locale.decode_region("BE"), "Belgium")
        self.assertEqual(web.locale.decode_region("??"), None)
        print("pattern.web.locale.decode_region()")

    def test_languages(self):
        # Assert "BE" => "fr" + "nl".
        self.assertEqual(web.locale.languages("be"), ["fr", "nl"])
        print("pattern.web.locale.languages()")

    def test_regions(self):
        # Assert "nl" => "NL" + "BE".
        self.assertEqual(web.locale.regions("nl"), ["NL", "BE"])
        print("pattern.web.locale.regions()")

    def test_regionalize(self):
        # Assert "nl" => "nl-NL" + "nl-BE".
        self.assertEqual(web.locale.regionalize("nl"), ["nl-NL", "nl-BE"])
        print("pattern.web.locale.regionalize()")

    def test_geocode(self):
        # Assert region geocode.
        v = web.locale.geocode("brussels")
        self.assertAlmostEqual(v[0], 50.83, places=2)
        self.assertAlmostEqual(v[1], 4.33, places=2)
        self.assertEqual(v[2], "nl")
        self.assertEqual(v[3], "Belgium")
        print("pattern.web.locale.geocode()")

    def test_correlation(self):
        # Test the correlation between locale.LANGUAGE_REGION and locale.GEOCODE.
        # It should increase as new languages and locations are added.
        i = 0
        n = len(web.locale.GEOCODE)
        for city, (latitude, longitude, language, region) in web.locale.GEOCODE.items():
            if web.locale.encode_region(region) is not None:
                i += 1
        self.assertTrue(float(i) / n > 0.60)

#---------------------------------------------------------------------------------------------------
# You need to define a username, password and mailbox to test on.


class TestMail(unittest.TestCase):

    def setUp(self):
        self.username = ""
        self.password = ""
        self.service = web.GMAIL
        self.port = 993
        self.SSL = True
        self.query1 = "google" # FROM-field query in Inbox.
        self.query2 = "viagra" # SUBJECT-field query in Spam.

    def test_mail(self):
        if not self.username or not self.password:
            return
        # Assert web.imap.Mail.
        m = web.Mail(self.username, self.password, service=self.service, port=self.port, secure=self.SSL)
        # Assert web.imap.MailFolder (assuming GMail folders).
        print(m.folders)
        self.assertTrue(len(m.folders) > 0)
        self.assertTrue(len(m.inbox) > 0)
        print("pattern.web.Mail")

    def test_mail_message1(self):
        if not self.username or not self.password or not self.query1:
            return
        # Assert web.imap.Mailfolder.search().
        m = web.Mail(self.username, self.password, service=self.service, port=self.port, secure=self.SSL)
        a = m.inbox.search(self.query1, field=web.FROM)
        self.assertTrue(isinstance(a[0], int))
        # Assert web.imap.Mailfolder.read().
        e = m.inbox.read(a[0], attachments=False, cached=False)
        # Assert web.imap.Message.
        self.assertTrue(isinstance(e, web.imap.Message))
        self.assertTrue(isinstance(e.author, str))
        self.assertTrue(isinstance(e.email_address, str))
        self.assertTrue(isinstance(e.date, str))
        self.assertTrue(isinstance(e.subject, str))
        self.assertTrue(isinstance(e.body, str))
        self.assertTrue(self.query1 in e.author.lower())
        self.assertTrue("@" in e.email_address)
        print("pattern.web.Mail.search(field=FROM)")
        print("pattern.web.Mail.read()")

    def test_mail_message2(self):
        if not self.username or not self.password or not self.query2:
            return
        # Test if we can download some mail attachments.
        # Set query2 to a mail subject of a spam e-mail you know contains an attachment.
        m = web.Mail(self.username, self.password, service=self.service, port=self.port, secure=self.SSL)
        if "spam" in m.folders:
            for id in m.spam.search(self.query2, field=web.SUBJECT):
                e = m.spam.read(id, attachments=True, cached=False)
                if len(e.attachments) > 0:
                    self.assertTrue(isinstance(e.attachments[0][1], bytes))
                    self.assertTrue(len(e.attachments[0][1]) > 0)
                    print("pattern.web.Message.attachments (MIME-type: %s)" % e.attachments[0][0])
        print("pattern.web.Mail.search(field=SUBJECT)")
        print("pattern.web.Mail.read()")

#---------------------------------------------------------------------------------------------------


class TestCrawler(unittest.TestCase):

    def setUp(self):
        pass

    def test_link(self):
        # Assert web.Link parser and properties.
        v = web.HTMLLinkParser().parse("""
            <html>
            <head>
                <title>title</title>
            </head>
            <body>
                <div id="navigation">
                    <a href="http://www.domain1.com/?p=1" title="1" rel="a">nav1</a>
                    <a href="http://www.domain2.com/?p=2" title="2" rel="b">nav1</a>
                </div>
            </body>
            </html>
        """, "http://www.domain.com/")
        self.assertTrue(v[0].url, "http://www.domain1.com/?p=1")
        self.assertTrue(v[1].url, "http://www.domain1.com/?p=2")
        self.assertTrue(v[0].description, "1")
        self.assertTrue(v[1].description, "2")
        self.assertTrue(v[0].relation, "a")
        self.assertTrue(v[1].relation, "b")
        self.assertTrue(v[0].referrer, "http://www.domain.com/")
        self.assertTrue(v[1].referrer, "http://www.domain.com/")
        self.assertTrue(v[0] < v[1])
        print("pattern.web.HTMLLinkParser")

    def test_crawler_crawl(self):
        # Assert domain filter.
        v = web.Crawler(links=["http://nodebox.net/"], domains=["nodebox.net"], delay=0.5)
        while len(v.visited) < 4:
            v.crawl(throttle=0.1, cached=False)
        for url in v.visited:
            self.assertTrue("nodebox.net" in url)
        self.assertTrue(len(v.history) == 2)
        print("pattern.web.Crawler.crawl()")

    def test_crawler_delay(self):
        # Assert delay for several crawls to a single domain.
        v = web.Crawler(links=["http://nodebox.net/"], domains=["nodebox.net"], delay=1.2)
        v.crawl()
        t = time.time()
        while not v.crawl(throttle=0.1, cached=False):
            pass
        t = time.time() - t
        self.assertTrue(t > 1.0)
        print("pattern.web.Crawler.delay")

    def test_crawler_breadth(self):
        # Assert BREADTH cross-domain preference.
        v = web.Crawler(links=["http://nodebox.net/"], delay=10)
        while len(v.visited) < 4:
            v.crawl(throttle=0.1, cached=False, method=web.BREADTH)
        self.assertTrue(list(v.history.keys())[0] != list(v.history.keys())[1])
        self.assertTrue(list(v.history.keys())[0] != list(v.history.keys())[2])
        self.assertTrue(list(v.history.keys())[1] != list(v.history.keys())[2])
        print("pattern.web.Crawler.crawl(method=BREADTH)")

#---------------------------------------------------------------------------------------------------


def suite():
    suite = unittest.TestSuite()
    suite.addTest(unittest.TestLoader().loadTestsFromTestCase(TestCache))
    suite.addTest(unittest.TestLoader().loadTestsFromTestCase(TestUnicode))
    suite.addTest(unittest.TestLoader().loadTestsFromTestCase(TestURL))
    suite.addTest(unittest.TestLoader().loadTestsFromTestCase(TestPlaintext))
    suite.addTest(unittest.TestLoader().loadTestsFromTestCase(TestSearchEngine))
    suite.addTest(unittest.TestLoader().loadTestsFromTestCase(TestDOM))
    suite.addTest(unittest.TestLoader().loadTestsFromTestCase(TestDocumentParser))
    suite.addTest(unittest.TestLoader().loadTestsFromTestCase(TestLocale))
    suite.addTest(unittest.TestLoader().loadTestsFromTestCase(TestMail))
    suite.addTest(unittest.TestLoader().loadTestsFromTestCase(TestCrawler))
    return suite

if __name__ == "__main__":

    result = unittest.TextTestRunner(verbosity=1).run(suite())
    sys.exit(not result.wasSuccessful())