parallel-library/vendor/iso639/iso639.py

# coding=utf-8
"""
Python library for ISO 639 standard

Copyright (c) 2014-2016 Mikael Karlsson (CSC - IT Center for Science Ltd.).
Licensed under AGPLv3.
"""

# Fix for Python 3.0 - 3.2
if not __package__:
    __package__ = __name__.split('.')[0]


def _fabtabular():
    """
    This function retrieves the ISO 639 and inverted names datasets as tsv files and returns them as lists.
    """
    import csv
    import sys
    from pkg_resources import resource_filename

    data = resource_filename(__package__, 'iso-639-3.tab')
    inverted = resource_filename(__package__, 'iso-639-3_Name_Index.tab')
    macro = resource_filename(__package__, 'iso-639-3-macrolanguages.tab')
    part5 = resource_filename(__package__, 'iso639-5.tsv')
    part2 = resource_filename(__package__, 'iso639-2.tsv')
    part1 = resource_filename(__package__, 'iso639-1.tsv')

    # if sys.version_info[0] == 2:
    #     from urllib2 import urlopen
    #     from contextlib import closing
    #     data_fo = closing(urlopen('http://www-01.sil.org/iso639-3/iso-639-3.tab'))
    #     inverted_fo = closing(urlopen('http://www-01.sil.org/iso639-3/iso-639-3_Name_Index.tab'))
    # else:
    #     from urllib.request import urlopen
    #     import io
    #     data_fo = io.StringIO(urlopen('http://www-01.sil.org/iso639-3/iso-639-3.tab').read().decode())
    #     inverted_fo = io.StringIO(urlopen('http://www-01.sil.org/iso639-3/iso-639-3_Name_Index.tab').read().decode())

    if sys.version_info[0] == 3:
        from functools import partial

        global open
        open = partial(open, encoding='utf-8')

    data_fo = open(data)
    inverted_fo = open(inverted)
    macro_fo = open(macro)
    part5_fo = open(part5)
    part2_fo = open(part2)
    part1_fo = open(part1)
    with data_fo as u:
        with inverted_fo as i:
            with macro_fo as m:
                with part5_fo as p5:
                    with part2_fo as p2:
                        with part1_fo as p1:
                            return (list(csv.reader(u, delimiter='\t'))[1:],
                                    list(csv.reader(i, delimiter='\t'))[1:],
                                    list(csv.reader(m, delimiter='\t'))[1:],
                                    list(csv.reader(p5, delimiter='\t'))[1:],
                                    list(csv.reader(p2, delimiter='\t'))[1:],
                                    list(csv.reader(p1, delimiter='\t'))[1:])


class _Language(object):
    """
    This class represents a language. It provides pycountry language class compatibility.
    """

    def __init__(self, part3, part2b, part2t, part1, name, inverted, macro, names, part5):
        self.part3 = part3
        self.part2b = part2b
        self.part2t = part2t
        self.part1 = part1
        self.name = name
        self.inverted = inverted
        self.macro = macro
        self.names = names
        self.part5 = part5

    def __getattr__(self, item):
        compat = {
            'alpha2': self.part1,
            'bibliographic': self.part2b,
            'terminology': self.part2t,
        }
        if item not in compat:
            raise AttributeError("'{o}' object has no attribute '{a}'".format(o=type(self).__name__, a=item))
        return compat[item]


class lazy_property(object):
    """
    Implements a lazy property decorator, that overwrites itself/property with value
    """

    def __init__(self, f):
        self.f = f
        self.name = f.__name__

    def __get__(self, instance, owner=None):
        if instance is None:
            return self
        val = self.f(instance)
        setattr(instance, self.name, val)
        return val


class Iso639(object):
    """
    This class is a close to drop-in replacement for pycountry.languages.
    But unlike pycountry.languages it also supports ISO 639-3.

    It implements the Singleton design pattern for performance reasons.
    Is uses lazy properties for faster import time.
    """

    def __new__(cls):
        if not hasattr(cls, '__instance'):
            setattr(cls, '__instance', super(cls, cls).__new__(cls))
        return getattr(cls, '__instance')

    def __len__(self):
        return len(self.languages)

    def __iter__(self):
        return iter(self.languages)

    def __getattr__(self, item):
        compat = {
            'alpha2': self.part1,
            'bibliographic': self.part2b,
            'terminology': self.part2t,
        }
        if item not in compat:
            raise AttributeError("'{o}' object has no attribute '{a}'".format(o=type(self).__name__, a=item))
        return compat[item]

    @lazy_property
    def languages(self):
        def generate():
            # All of part3 and matching part2
            for a, b, c, d, _, _, e, _ in l:
                inv = alt[a].pop(e)
                yield _Language(a, b, c,
                                d if d in p1c else '',  # Fixes 'sh'
                                e, inv,
                                m.get(a, [''])[0],
                                list(alt[a].items()),
                                '')
                p2.pop(b, None)
                p2.pop(c, None)

            # All of part5 and matching part2
            for _, a, b, _ in p5:
                yield _Language('',
                                a if a in p2 else '',
                                a if a in p2 else '',
                                p1n.get(b, ['', ''])[1],
                                b, '', '', '', a)
                p2.pop(a, None)

            # Rest of part2
            p2.pop('qaa-qtz', None)  # Is not a real code, but a range
            for _, a, b, _ in p2.values():
                n = [x.strip() for x in b.split('|')]
                yield _Language('', a, a,
                                p1n.get(b, ['', ''])[1],
                                n[0], '', '', zip(n[1:], n[1:]), '')

        import collections

        l, i, m, p5, p2, p1 = _fabtabular()
        alt = collections.defaultdict(dict)
        for x in i:
            alt[x[0]][x[1]] = x[2]
        m = dict((x[1], x) for x in m)
        p2 = dict((x[1], x) for x in p2)
        p1c = dict((x[1], x) for x in p1)
        p1n = dict((x[2].split('|')[0].strip(), x) for x in p1)
        return list(generate())

    @lazy_property
    def part3(self):
        return dict((x.part3, x) for x in self.languages if x.part3)

    @lazy_property
    def part2b(self):
        return dict((x.part2b, x) for x in self.languages if x.part2b)

    @lazy_property
    def part2t(self):
        return dict((x.part2t, x) for x in self.languages if x.part2t)

    @lazy_property
    def part1(self):
        return dict((x.part1, x) for x in self.languages if x.part1)

    @lazy_property
    def part5(self):
        return dict((x.part5, x) for x in self.languages if x.part5)

    @lazy_property
    def name(self):
        def gen():
            for x in self.languages:
                if x.name:
                    yield x.name, x
                for n in x.names:
                    yield n[0], x

        return dict(gen())

    @lazy_property
    def inverted(self):
        return dict((x.inverted, x) for x in self.languages if x.inverted)

    @lazy_property
    def macro(self):
        import collections

        m = collections.defaultdict(list)
        for x in self.languages:
            if x.macro:
                m[x.macro].append(x)
        return dict(m)

    @lazy_property
    def retired(self):
        """
        Function for generating retired languages. Returns a dict('code', (datetime, [language, ...], 'description')).
        """

        def gen():
            import csv
            import re
            from datetime import datetime
            from pkg_resources import resource_filename

            with open(resource_filename(__package__, 'iso-639-3_Retirements.tab')) as rf:
                rtd = list(csv.reader(rf, delimiter='\t'))[1:]
                rc = [r[0] for r in rtd]
                for i, _, _, m, s, d in rtd:
                    d = datetime.strptime(d, '%Y-%m-%d')
                    if not m:
                        m = re.findall('\[([a-z]{3})\]', s)
                    if m:
                        m = [m] if isinstance(m, str) else m
                        yield i, (d, [self.get(part3=x) for x in m if x not in rc], s)
                    else:
                        yield i, (d, [], s)

            yield 'sh', self.get(part3='hbs')  # Add 'sh' as deprecated

        return dict(gen())

    def get(self, **kwargs):
        """
        Simple getter function for languages. Takes 1 keyword/value and returns 1 language object.
        """
        if not len(kwargs) == 1:
            raise AttributeError('Only one keyword expected')
        key, value = kwargs.popitem()
        return getattr(self, key)[value]