i18n/polib.py

43075:57875cf423c9

from __future__ import absolute_import

__author__ = 'David Jean Louis <[e-mail]>'
__version__ = '1.0.7'
__all__ = ['pofile', 'POFile', 'POEntry', 'mofile', 'MOFile', 'MOEntry',
           'default_encoding', 'escape', 'unescape', 'detect_encoding', ]

import array
import codecs
import os
import re

        return s

    def u(s):
        return unicode(s, "unicode_escape")

else:
    PY3 = True
    text_type = str

    def b(s):
        return s.encode("latin-1")

    def u(s):
        return s
# }}}
# _pofile_or_mofile {{{


def _pofile_or_mofile(f, type, **kwargs):

    kls = type == 'pofile' and _POFileParser or _MOFileParser
    parser = kls(
        f,
        encoding=enc,
        check_for_duplicates=kwargs.get('check_for_duplicates', False),
        klass=kwargs.get('klass')
    )
    instance = parser.parse()
    instance.wrapwidth = kwargs.get('wrapwidth', 78)
    return instance
# }}}
# _is_file {{{


def _is_file(filename_or_contents):

    """
    try:
        return os.path.exists(filename_or_contents)
    except (ValueError, UnicodeEncodeError):
        return False
# }}}
# function pofile() {{{


def pofile(pofile, **kwargs):

        class which is used to instantiate the return value (optional,
        default: ``None``, the return value with be a :class:`~polib.POFile`
        instance).
    """
    return _pofile_or_mofile(pofile, 'pofile', **kwargs)
# }}}
# function mofile() {{{


def mofile(mofile, **kwargs):

        class which is used to instantiate the return value (optional,
        default: ``None``, the return value with be a :class:`~polib.POFile`
        instance).
    """
    return _pofile_or_mofile(mofile, 'mofile', **kwargs)
# }}}
# function detect_encoding() {{{


def detect_encoding(file, binary_mode=False):

                    enc = enc.decode('utf-8')
                if charset_exists(enc):
                    return enc
        f.close()
    return default_encoding
# }}}
# function escape() {{{


def escape(st):
    """
    Escapes the characters ``\\\\``, ``\\t``, ``\\n``, ``\\r`` and ``"`` in
    the given string ``st`` and returns it.
    """
    return st.replace('\\', r'\\')\
             .replace('\t', r'\t')\
             .replace('\r', r'\r')\
             .replace('\n', r'\n')\
             .replace('\"', r'\"')
# }}}
# function unescape() {{{


def unescape(st):
    """
    Unescapes the characters ``\\\\``, ``\\t``, ``\\n``, ``\\r`` and ``"`` in
    the given string ``st`` and returns it.
    """
    def unescape_repl(m):
        m = m.group(1)
        if m == 'n':
            return '\n'
        if m == 't':

        if m == 'r':
            return '\r'
        if m == '\\':
            return '\\'
        return m  # handles escaped double quote
    return re.sub(r'\\(\\|n|t|r|")', unescape_repl, st)
# }}}
# class _BaseFile {{{


class _BaseFile(list):

    def __unicode__(self):
        """
        Returns the unicode representation of the file.
        """
        ret = []
        entries = [self.metadata_as_entry()] + \
                  [e for e in self if not e.obsolete]
        for entry in entries:
            ret.append(entry.__unicode__(self.wrapwidth))
        for entry in self.obsolete_entries():
            ret.append(entry.__unicode__(self.wrapwidth))
        ret = u('\n').join(ret)

        assert isinstance(ret, text_type)
        #if type(ret) != text_type:
        #    return unicode(ret, self.encoding)
        return ret

    if PY3:
        def __str__(self):
            return self.__unicode__()
    else:
        def __str__(self):
            """
            Returns the string representation of the file.
            """
            return unicode(self).encode(self.encoding)

        Argument:

        ``entry``
            an instance of :class:`~polib._BaseEntry`.
        """
        return self.find(entry.msgid, by='msgid', msgctxt=entry.msgctxt) \
            is not None

    def __eq__(self, other):
        return str(self) == str(other)

    def append(self, entry):

        fhandle.close()
        # set the file path if not set
        if self.fpath is None and fpath:
            self.fpath = fpath

    def find(self, st, by='msgid', include_obsolete_entries=False,
             msgctxt=False):
        """
        Find the entry which msgid (or property identified by the ``by``
        argument) matches the string ``st``.

        Keyword arguments:

            'Language-Team',
            'MIME-Version',
            'Content-Type',
            'Content-Transfer-Encoding',
            'Language',
            'Plural-Forms'
        ]
        ordered_data = []
        for data in data_order:
            try:
                value = metadata.pop(data)

                return 1
            elif self_msgid < other_msgid:
                return -1
            else:
                return 0
        # add metadata entry
        entries.sort(key=lambda o: o.msgctxt or o.msgid)
        mentry = self.metadata_as_entry()
        #mentry.msgstr = mentry.msgstr.replace('\\n', '').lstrip()
        entries = [mentry] + entries
        entries_len = len(entries)
        ids, strs = b(''), b('')
        for e in entries:
            # For each string, we need size and file offset.  Each string is

            # start of key index
            7 * 4,
            # start of value index
            7 * 4 + entries_len * 8,
            # size and offset of hash table, we don't use hash tables
            0, keystart

        )
        if PY3 and sys.version_info.minor > 1:  # python 3.2 or superior
            output += array.array("i", offsets).tobytes()
        else:
            output += array.array("i", offsets).tostring()

        only if it's an unicode string and returns the encoded string.
        """
        if isinstance(mixed, text_type):
            mixed = mixed.encode(self.encoding)
        return mixed
# }}}
# class POFile {{{


class POFile(_BaseFile):

    def untranslated_entries(self):
        """
        Convenience method that returns the list of untranslated entries.
        """
        return [e for e in self if not e.translated() and not e.obsolete
                and not 'fuzzy' in e.flags]

    def fuzzy_entries(self):
        """
        Convenience method that returns the list of fuzzy entries.
        """

            e.merge(entry)
        # ok, now we must "obsolete" entries that are not in the refpot anymore
        for entry in self:
            if entry.msgid not in refpot_msgids:
                entry.obsolete = True
# }}}
# class MOFile {{{


class MOFile(_BaseFile):
    """
    Mo file reader/writer.
    This class inherits the :class:`~polib._BaseFile` class and, by
    extension, the python ``list`` type.
    """
    MAGIC = 0x950412de
    MAGIC_SWAPPED = 0xde120495

    def __init__(self, *args, **kwargs):
        """
        Constructor, accepts all keywords arguments accepted by
        :class:`~polib._BaseFile` class.

    def obsolete_entries(self):
        """
        Convenience method to keep the same interface with POFile instances.
        """
        return []
# }}}
# class _BaseEntry {{{


class _BaseEntry(object):

        else:
            delflag = ''
        ret = []
        # write the msgctxt if any
        if self.msgctxt is not None:
            ret += self._str_field("msgctxt", delflag, "", self.msgctxt,
                                   wrapwidth)
        # write the msgid
        ret += self._str_field("msgid", delflag, "", self.msgid, wrapwidth)
        # write the msgid_plural if any
        if self.msgid_plural:
            ret += self._str_field("msgid_plural", delflag, "",
                                   self.msgid_plural, wrapwidth)
        if self.msgstr_plural:
            # write the msgstr_plural if any
            msgstrs = self.msgstr_plural
            keys = list(msgstrs)
            keys.sort()
            for index in keys:
                msgstr = msgstrs[index]
                plural_index = '[%s]' % index
                ret += self._str_field("msgstr", delflag, plural_index, msgstr,
                                       wrapwidth)
        else:
            # otherwise write the msgstr
            ret += self._str_field("msgstr", delflag, "", self.msgstr,
                                   wrapwidth)
        ret.append('')
        ret = u('\n').join(ret)
        return ret

    if PY3:
        def __str__(self):
            return self.__unicode__()
    else:
        def __str__(self):
            """
            Returns the string representation of the entry.
            """
            return unicode(self).encode(self.encoding)

    def __eq__(self, other):
        return str(self) == str(other)

    def _str_field(self, fieldname, delflag, plural_index, field,
                   wrapwidth=78):
        lines = field.splitlines(True)
        if len(lines) > 1:
            lines = [''] + lines  # start with initial empty line
        else:
            escaped_field = escape(field)

            if plural_index:
                flength += len(plural_index)
            real_wrapwidth = wrapwidth - flength + specialchars_count
            if wrapwidth > 0 and len(field) > real_wrapwidth:
                # Wrap the line but take field name into account
                lines = [''] + [unescape(item) for item in wrap(
                    escaped_field,
                    wrapwidth - 2,  # 2 for quotes ""
                    drop_whitespace=False,
                    break_long_words=False
                )]
            else:
                lines = [field]
        if fieldname.startswith('previous_'):
            # quick and dirty trick to get the real field name
            fieldname = fieldname[9:]

        ret = ['%s%s%s "%s"' % (delflag, fieldname, plural_index,
                                escape(lines.pop(0)))]
        for line in lines:
            ret.append('%s"%s"' % (delflag, escape(line)))
        return ret
# }}}
# class POEntry {{{


class POEntry(_BaseEntry):

                        ret += wrap(
                            comment,
                            wrapwidth,
                            initial_indent=c[1],
                            subsequent_indent=c[1],
                            break_long_words=False
                        )
                    else:
                        ret.append('%s%s' % (c[1], comment))

        # occurrences (with text wrapping as xgettext does)

            if wrapwidth > 0 and len(filestr) + 3 > wrapwidth:
                # textwrap split words that contain hyphen, this is not
                # what we want for filenames, so the dirty hack is to
                # temporally replace hyphens with a char that a file cannot
                # contain, like "*"
                ret += [l.replace('*', '-') for l in wrap(
                    filestr.replace('-', '*'),
                    wrapwidth,
                    initial_indent='#: ',
                    subsequent_indent='#: ',
                    break_long_words=False
                )]
            else:
                ret.append('#: ' + filestr)

        # flags (TODO: wrapping ?)
        if self.flags:
            ret.append('#, %s' % ', '.join(self.flags))

        # previous context and previous msgid/msgid_plural
        fields = ['previous_msgctxt', 'previous_msgid',
                  'previous_msgid_plural']
        for f in fields:
            val = getattr(self, f)
            if val:
                ret += self._str_field(f, "#| ", "", val, wrapwidth)

        ret.append(_BaseEntry.__unicode__(self, wrapwidth))
        ret = u('\n').join(ret)

        assert isinstance(ret, text_type)
        #if type(ret) != types.UnicodeType:
        #    return unicode(ret, self.encoding)
        return ret

    def __cmp__(self, other):
        """

                except KeyError:
                    self.msgstr_plural[pos] = ''

    def __hash__(self):
        return hash((self.msgid, self.msgstr))
# }}}
# class MOEntry {{{


class MOEntry(_BaseEntry):
    """
    Represents a mo file entry.
    """
    def __init__(self, *args, **kwargs):
        """
        Constructor, accepts the following keyword arguments,
        for consistency with :class:`~polib.POEntry`:

        self.previous_msgid = None
        self.previous_msgid_plural = None

    def __hash__(self):
        return hash((self.msgid, self.msgstr))

# }}}
# class _POFileParser {{{

        if klass is None:
            klass = POFile
        self.instance = klass(
            pofile=pofile,
            encoding=enc,
            check_for_duplicates=kwargs.get('check_for_duplicates', False)
        )
        self.transitions = {}
        self.current_line = 0
        self.current_entry = POEntry(linenum=self.current_line)
        self.current_state = 'st'

        #     * MI: a msgid
        #     * MP: a msgid plural
        #     * MS: a msgstr
        #     * MX: a msgstr plural
        #     * MC: a msgid or msgstr continuation line
        all = ['st', 'he', 'gc', 'oc', 'fl', 'ct', 'pc', 'pm', 'pp', 'tc',
               'ms', 'mp', 'mx', 'mi']

        self.add('tc', ['st', 'he'],                                     'he')
        self.add('tc', ['gc', 'oc', 'fl', 'tc', 'pc', 'pm', 'pp', 'ms',
                        'mp', 'mx', 'mi'],                               'tc')
        self.add('gc', all,                                              'gc')
        self.add('oc', all,                                              'oc')
        self.add('fl', all,                                              'fl')
        self.add('pc', all,                                              'pc')
        self.add('pm', all,                                              'pm')
        self.add('pp', all,                                              'pp')
        self.add('ct', ['st', 'he', 'gc', 'oc', 'fl', 'tc', 'pc', 'pm',
                        'pp', 'ms', 'mx'],                               'ct')
        self.add('mi', ['st', 'he', 'gc', 'oc', 'fl', 'ct', 'tc', 'pc',
                 'pm', 'pp', 'ms', 'mx'],                                'mi')
        self.add('mp', ['tc', 'gc', 'pc', 'pm', 'pp', 'mi'],             'mp')
        self.add('ms', ['mi', 'mp', 'tc'],                               'ms')
        self.add('mx', ['mi', 'mx', 'mp', 'tc'],                         'mx')
        self.add('mc', ['ct', 'mi', 'mp', 'ms', 'mx', 'pm', 'pp', 'pc'], 'mc')

    def parse(self):
        """
        Run the state machine, parse the file line by line and call process()

                self.entry_obsolete = 0

            # Take care of keywords like
            # msgid, msgid_plural, msgctxt & msgstr.
            if tokens[0] in keywords and nb_tokens > 1:
                line = line[len(tokens[0]):].lstrip()
                if re.search(r'([^\\]|^)"', line[1:-1]):
                    raise IOError('Syntax error in po file %s (line %s): '
                                  'unescaped double quote found' %
                                  (self.instance.fpath, self.current_line))
                self.current_token = line
                self.process(keywords[tokens[0]])
                continue

            self.current_token = line

                self.process('oc')

            elif line[:1] == '"':
                # we are on a continuation line
                if re.search(r'([^\\]|^)"', line[1:-1]):
                    raise IOError('Syntax error in po file %s (line %s): '
                                  'unescaped double quote found' %
                                  (self.instance.fpath, self.current_line))
                self.process('mc')

            elif line[:7] == 'msgstr[':
                # we are on a msgstr plural
                self.process('mx')

                # we are on a generated comment line
                self.process('gc')

            elif tokens[0] == '#|':
                if nb_tokens <= 1:
                    raise IOError('Syntax error in po file %s (line %s)' %
                                  (self.instance.fpath, self.current_line))

                # Remove the marker and any whitespace right after that.
                line = line[2:].lstrip()
                self.current_token = line

                    self.process('mc')
                    continue

                if nb_tokens == 2:
                    # Invalid continuation line.
                    raise IOError('Syntax error in po file %s (line %s): '
                                  'invalid continuation line' %
                                  (self.instance.fpath, self.current_line))

                # we are on a "previous translation" comment line,
                if tokens[1] not in prev_keywords:
                    # Unknown keyword in previous translation comment.
                    raise IOError('Syntax error in po file %s (line %s): '
                                  'unknown keyword %s' %
                                  (self.instance.fpath, self.current_line,
                                   tokens[1]))

                # Remove the keyword and any whitespace
                # between it and the starting quote.
                line = line[len(tokens[1]):].lstrip()
                self.current_token = line
                self.process(prev_keywords[tokens[1]])

            else:
                raise IOError('Syntax error in po file %s (line %s)' %
                              (self.instance.fpath, self.current_line))

        if self.current_entry and len(tokens) > 0 and \
           not tokens[0].startswith('#'):
            # since entries are added when another entry is found, we must add
            # the last entry here (only if there are lines). Trailing comments
            # are ignored
            self.instance.append(self.current_entry)

        try:
            (action, state) = self.transitions[(symbol, self.current_state)]
            if action():
                self.current_state = state
        except Exception:
            raise IOError('Syntax error in po file (line %s)' %
                          self.current_line)

    # state handlers

    def handle_he(self):
        """Handle a header comment."""

    def handle_fl(self):
        """Handle a flags line."""
        if self.current_state in ['mc', 'ms', 'mx']:
            self.instance.append(self.current_entry)
            self.current_entry = POEntry(linenum=self.current_line)
        self.current_entry.flags += [c.strip() for c in
                                     self.current_token[3:].split(',')]
        return True

    def handle_pp(self):
        """Handle a previous msgid_plural line."""
        if self.current_state in ['mc', 'ms', 'mx']:
            self.instance.append(self.current_entry)
            self.current_entry = POEntry(linenum=self.current_line)
        self.current_entry.previous_msgid_plural = \
            unescape(self.current_token[1:-1])
        return True

    def handle_pm(self):
        """Handle a previous msgid line."""
        if self.current_state in ['mc', 'ms', 'mx']:
            self.instance.append(self.current_entry)
            self.current_entry = POEntry(linenum=self.current_line)
        self.current_entry.previous_msgid = \
            unescape(self.current_token[1:-1])
        return True

    def handle_pc(self):
        """Handle a previous msgctxt line."""
        if self.current_state in ['mc', 'ms', 'mx']:
            self.instance.append(self.current_entry)
            self.current_entry = POEntry(linenum=self.current_line)
        self.current_entry.previous_msgctxt = \
            unescape(self.current_token[1:-1])
        return True

    def handle_ct(self):
        """Handle a msgctxt."""
        if self.current_state in ['mc', 'ms', 'mx']:

        return True

    def handle_mx(self):
        """Handle a msgstr plural."""
        index = self.current_token[7]
        value = self.current_token[self.current_token.find('"') + 1:-1]
        self.current_entry.msgstr_plural[int(index)] = unescape(value)
        self.msgstr_index = int(index)
        return True

    def handle_mc(self):

            self.current_entry.previous_msgid += token
        elif self.current_state == 'pc':
            self.current_entry.previous_msgctxt += token
        # don't change the current state
        return False
# }}}
# class _MOFileParser {{{


class _MOFileParser(object):

        if klass is None:
            klass = MOFile
        self.instance = klass(
            fpath=mofile,
            encoding=kwargs.get('encoding', default_encoding),
            check_for_duplicates=kwargs.get('check_for_duplicates', False)
        )

    def __del__(self):
        """
        Make sure the file is closed, this prevents warnings on unclosed file

            msgid_tokens = msgid.split(b('\0'))
            if len(msgid_tokens) > 1:
                entry = self._build_entry(
                    msgid=msgid_tokens[0],
                    msgid_plural=msgid_tokens[1],
                    msgstr_plural=dict((k, v) for k, v in
                                       enumerate(msgstr.split(b('\0'))))
                )
            else:
                entry = self._build_entry(msgid=msgid, msgstr=msgstr)
            self.instance.append(entry)
        # close opened file
        self.fhandle.close()
        return self.instance

    def _build_entry(self, msgid, msgstr=None, msgid_plural=None,
                     msgstr_plural=None):
        msgctxt_msgid = msgid.split(b('\x04'))
        encoding = self.instance.encoding
        if len(msgctxt_msgid) > 1:
            kwargs = {
                'msgctxt': msgctxt_msgid[0].decode(encoding),

        bytes = self.fhandle.read(numbytes)
        tup = struct.unpack(fmt, bytes)
        if len(tup) == 1:
            return tup[0]
        return tup
# }}}
# class TextWrapper {{{


class TextWrapper(textwrap.TextWrapper):
    """
    Subclass of textwrap.TextWrapper that backport the
    drop_whitespace option.
    """
    def __init__(self, *args, **kwargs):
        drop_whitespace = kwargs.pop('drop_whitespace', True)
        textwrap.TextWrapper.__init__(self, *args, **kwargs)
        self.drop_whitespace = drop_whitespace

            # of all lines (return value).
            if cur_line:
                lines.append(indent + ''.join(cur_line))

        return lines
# }}}
# function wrap() {{{


def wrap(text, width=70, **kwargs):

    """
    if sys.version_info < (2, 6):
        return TextWrapper(width=width, **kwargs).wrap(text)
    return textwrap.wrap(text, width=width, **kwargs)

# }}}

43076:2372284d9457

from __future__ import absolute_import

__author__ = 'David Jean Louis <[e-mail]>'
__version__ = '1.0.7'
__all__ = [
    'pofile',
    'POFile',
    'POEntry',
    'mofile',
    'MOFile',
    'MOEntry',
    'default_encoding',
    'escape',
    'unescape',
    'detect_encoding',
]

import array
import codecs
import os
import re

        return s

    def u(s):
        return unicode(s, "unicode_escape")


else:
    PY3 = True
    text_type = str

    def b(s):
        return s.encode("latin-1")

    def u(s):
        return s


# }}}
# _pofile_or_mofile {{{


def _pofile_or_mofile(f, type, **kwargs):

    kls = type == 'pofile' and _POFileParser or _MOFileParser
    parser = kls(
        f,
        encoding=enc,
        check_for_duplicates=kwargs.get('check_for_duplicates', False),
        klass=kwargs.get('klass'),
    )
    instance = parser.parse()
    instance.wrapwidth = kwargs.get('wrapwidth', 78)
    return instance


# }}}
# _is_file {{{


def _is_file(filename_or_contents):

    """
    try:
        return os.path.exists(filename_or_contents)
    except (ValueError, UnicodeEncodeError):
        return False


# }}}
# function pofile() {{{


def pofile(pofile, **kwargs):

        class which is used to instantiate the return value (optional,
        default: ``None``, the return value with be a :class:`~polib.POFile`
        instance).
    """
    return _pofile_or_mofile(pofile, 'pofile', **kwargs)


# }}}
# function mofile() {{{


def mofile(mofile, **kwargs):

        class which is used to instantiate the return value (optional,
        default: ``None``, the return value with be a :class:`~polib.POFile`
        instance).
    """
    return _pofile_or_mofile(mofile, 'mofile', **kwargs)


# }}}
# function detect_encoding() {{{


def detect_encoding(file, binary_mode=False):

                    enc = enc.decode('utf-8')
                if charset_exists(enc):
                    return enc
        f.close()
    return default_encoding


# }}}
# function escape() {{{


def escape(st):
    """
    Escapes the characters ``\\\\``, ``\\t``, ``\\n``, ``\\r`` and ``"`` in
    the given string ``st`` and returns it.
    """
    return (
        st.replace('\\', r'\\')
        .replace('\t', r'\t')
        .replace('\r', r'\r')
        .replace('\n', r'\n')
        .replace('\"', r'\"')
    )


# }}}
# function unescape() {{{


def unescape(st):
    """
    Unescapes the characters ``\\\\``, ``\\t``, ``\\n``, ``\\r`` and ``"`` in
    the given string ``st`` and returns it.
    """

    def unescape_repl(m):
        m = m.group(1)
        if m == 'n':
            return '\n'
        if m == 't':

        if m == 'r':
            return '\r'
        if m == '\\':
            return '\\'
        return m  # handles escaped double quote

    return re.sub(r'\\(\\|n|t|r|")', unescape_repl, st)


# }}}
# class _BaseFile {{{


class _BaseFile(list):

    def __unicode__(self):
        """
        Returns the unicode representation of the file.
        """
        ret = []
        entries = [self.metadata_as_entry()] + [
            e for e in self if not e.obsolete
        ]
        for entry in entries:
            ret.append(entry.__unicode__(self.wrapwidth))
        for entry in self.obsolete_entries():
            ret.append(entry.__unicode__(self.wrapwidth))
        ret = u('\n').join(ret)

        assert isinstance(ret, text_type)
        # if type(ret) != text_type:
        #    return unicode(ret, self.encoding)
        return ret

    if PY3:

        def __str__(self):
            return self.__unicode__()

    else:

        def __str__(self):
            """
            Returns the string representation of the file.
            """
            return unicode(self).encode(self.encoding)

        Argument:

        ``entry``
            an instance of :class:`~polib._BaseEntry`.
        """
        return (
            self.find(entry.msgid, by='msgid', msgctxt=entry.msgctxt)
            is not None
        )

    def __eq__(self, other):
        return str(self) == str(other)

    def append(self, entry):

        fhandle.close()
        # set the file path if not set
        if self.fpath is None and fpath:
            self.fpath = fpath

    def find(
        self, st, by='msgid', include_obsolete_entries=False, msgctxt=False
    ):
        """
        Find the entry which msgid (or property identified by the ``by``
        argument) matches the string ``st``.

        Keyword arguments:

            'Language-Team',
            'MIME-Version',
            'Content-Type',
            'Content-Transfer-Encoding',
            'Language',
            'Plural-Forms',
        ]
        ordered_data = []
        for data in data_order:
            try:
                value = metadata.pop(data)

                return 1
            elif self_msgid < other_msgid:
                return -1
            else:
                return 0

        # add metadata entry
        entries.sort(key=lambda o: o.msgctxt or o.msgid)
        mentry = self.metadata_as_entry()
        # mentry.msgstr = mentry.msgstr.replace('\\n', '').lstrip()
        entries = [mentry] + entries
        entries_len = len(entries)
        ids, strs = b(''), b('')
        for e in entries:
            # For each string, we need size and file offset.  Each string is

            # start of key index
            7 * 4,
            # start of value index
            7 * 4 + entries_len * 8,
            # size and offset of hash table, we don't use hash tables
            0,
            keystart,
        )
        if PY3 and sys.version_info.minor > 1:  # python 3.2 or superior
            output += array.array("i", offsets).tobytes()
        else:
            output += array.array("i", offsets).tostring()

        only if it's an unicode string and returns the encoded string.
        """
        if isinstance(mixed, text_type):
            mixed = mixed.encode(self.encoding)
        return mixed


# }}}
# class POFile {{{


class POFile(_BaseFile):

    def untranslated_entries(self):
        """
        Convenience method that returns the list of untranslated entries.
        """
        return [
            e
            for e in self
            if not e.translated() and not e.obsolete and not 'fuzzy' in e.flags
        ]

    def fuzzy_entries(self):
        """
        Convenience method that returns the list of fuzzy entries.
        """

            e.merge(entry)
        # ok, now we must "obsolete" entries that are not in the refpot anymore
        for entry in self:
            if entry.msgid not in refpot_msgids:
                entry.obsolete = True


# }}}
# class MOFile {{{


class MOFile(_BaseFile):
    """
    Mo file reader/writer.
    This class inherits the :class:`~polib._BaseFile` class and, by
    extension, the python ``list`` type.
    """

    MAGIC = 0x950412DE
    MAGIC_SWAPPED = 0xDE120495

    def __init__(self, *args, **kwargs):
        """
        Constructor, accepts all keywords arguments accepted by
        :class:`~polib._BaseFile` class.

    def obsolete_entries(self):
        """
        Convenience method to keep the same interface with POFile instances.
        """
        return []


# }}}
# class _BaseEntry {{{


class _BaseEntry(object):

        else:
            delflag = ''
        ret = []
        # write the msgctxt if any
        if self.msgctxt is not None:
            ret += self._str_field(
                "msgctxt", delflag, "", self.msgctxt, wrapwidth
            )
        # write the msgid
        ret += self._str_field("msgid", delflag, "", self.msgid, wrapwidth)
        # write the msgid_plural if any
        if self.msgid_plural:
            ret += self._str_field(
                "msgid_plural", delflag, "", self.msgid_plural, wrapwidth
            )
        if self.msgstr_plural:
            # write the msgstr_plural if any
            msgstrs = self.msgstr_plural
            keys = list(msgstrs)
            keys.sort()
            for index in keys:
                msgstr = msgstrs[index]
                plural_index = '[%s]' % index
                ret += self._str_field(
                    "msgstr", delflag, plural_index, msgstr, wrapwidth
                )
        else:
            # otherwise write the msgstr
            ret += self._str_field(
                "msgstr", delflag, "", self.msgstr, wrapwidth
            )
        ret.append('')
        ret = u('\n').join(ret)
        return ret

    if PY3:

        def __str__(self):
            return self.__unicode__()

    else:

        def __str__(self):
            """
            Returns the string representation of the entry.
            """
            return unicode(self).encode(self.encoding)

    def __eq__(self, other):
        return str(self) == str(other)

    def _str_field(self, fieldname, delflag, plural_index, field, wrapwidth=78):
        lines = field.splitlines(True)
        if len(lines) > 1:
            lines = [''] + lines  # start with initial empty line
        else:
            escaped_field = escape(field)

            if plural_index:
                flength += len(plural_index)
            real_wrapwidth = wrapwidth - flength + specialchars_count
            if wrapwidth > 0 and len(field) > real_wrapwidth:
                # Wrap the line but take field name into account
                lines = [''] + [
                    unescape(item)
                    for item in wrap(
                        escaped_field,
                        wrapwidth - 2,  # 2 for quotes ""
                        drop_whitespace=False,
                        break_long_words=False,
                    )
                ]
            else:
                lines = [field]
        if fieldname.startswith('previous_'):
            # quick and dirty trick to get the real field name
            fieldname = fieldname[9:]

        ret = [
            '%s%s%s "%s"'
            % (delflag, fieldname, plural_index, escape(lines.pop(0)))
        ]
        for line in lines:
            ret.append('%s"%s"' % (delflag, escape(line)))
        return ret


# }}}
# class POEntry {{{


class POEntry(_BaseEntry):

                        ret += wrap(
                            comment,
                            wrapwidth,
                            initial_indent=c[1],
                            subsequent_indent=c[1],
                            break_long_words=False,
                        )
                    else:
                        ret.append('%s%s' % (c[1], comment))

        # occurrences (with text wrapping as xgettext does)

            if wrapwidth > 0 and len(filestr) + 3 > wrapwidth:
                # textwrap split words that contain hyphen, this is not
                # what we want for filenames, so the dirty hack is to
                # temporally replace hyphens with a char that a file cannot
                # contain, like "*"
                ret += [
                    l.replace('*', '-')
                    for l in wrap(
                        filestr.replace('-', '*'),
                        wrapwidth,
                        initial_indent='#: ',
                        subsequent_indent='#: ',
                        break_long_words=False,
                    )
                ]
            else:
                ret.append('#: ' + filestr)

        # flags (TODO: wrapping ?)
        if self.flags:
            ret.append('#, %s' % ', '.join(self.flags))

        # previous context and previous msgid/msgid_plural
        fields = ['previous_msgctxt', 'previous_msgid', 'previous_msgid_plural']
        for f in fields:
            val = getattr(self, f)
            if val:
                ret += self._str_field(f, "#| ", "", val, wrapwidth)

        ret.append(_BaseEntry.__unicode__(self, wrapwidth))
        ret = u('\n').join(ret)

        assert isinstance(ret, text_type)
        # if type(ret) != types.UnicodeType:
        #    return unicode(ret, self.encoding)
        return ret

    def __cmp__(self, other):
        """

                except KeyError:
                    self.msgstr_plural[pos] = ''

    def __hash__(self):
        return hash((self.msgid, self.msgstr))


# }}}
# class MOEntry {{{


class MOEntry(_BaseEntry):
    """
    Represents a mo file entry.
    """

    def __init__(self, *args, **kwargs):
        """
        Constructor, accepts the following keyword arguments,
        for consistency with :class:`~polib.POEntry`:

        self.previous_msgid = None
        self.previous_msgid_plural = None

    def __hash__(self):
        return hash((self.msgid, self.msgstr))


# }}}
# class _POFileParser {{{

        if klass is None:
            klass = POFile
        self.instance = klass(
            pofile=pofile,
            encoding=enc,
            check_for_duplicates=kwargs.get('check_for_duplicates', False),
        )
        self.transitions = {}
        self.current_line = 0
        self.current_entry = POEntry(linenum=self.current_line)
        self.current_state = 'st'

        #     * MI: a msgid
        #     * MP: a msgid plural
        #     * MS: a msgstr
        #     * MX: a msgstr plural
        #     * MC: a msgid or msgstr continuation line
        all = [
            'st',
            'he',
            'gc',
            'oc',
            'fl',
            'ct',
            'pc',
            'pm',
            'pp',
            'tc',
            'ms',
            'mp',
            'mx',
            'mi',
        ]

        self.add('tc', ['st', 'he'], 'he')
        self.add(
            'tc',
            ['gc', 'oc', 'fl', 'tc', 'pc', 'pm', 'pp', 'ms', 'mp', 'mx', 'mi'],
            'tc',
        )
        self.add('gc', all, 'gc')
        self.add('oc', all, 'oc')
        self.add('fl', all, 'fl')
        self.add('pc', all, 'pc')
        self.add('pm', all, 'pm')
        self.add('pp', all, 'pp')
        self.add(
            'ct',
            ['st', 'he', 'gc', 'oc', 'fl', 'tc', 'pc', 'pm', 'pp', 'ms', 'mx'],
            'ct',
        )
        self.add(
            'mi',
            [
                'st',
                'he',
                'gc',
                'oc',
                'fl',
                'ct',
                'tc',
                'pc',
                'pm',
                'pp',
                'ms',
                'mx',
            ],
            'mi',
        )
        self.add('mp', ['tc', 'gc', 'pc', 'pm', 'pp', 'mi'], 'mp')
        self.add('ms', ['mi', 'mp', 'tc'], 'ms')
        self.add('mx', ['mi', 'mx', 'mp', 'tc'], 'mx')
        self.add('mc', ['ct', 'mi', 'mp', 'ms', 'mx', 'pm', 'pp', 'pc'], 'mc')

    def parse(self):
        """
        Run the state machine, parse the file line by line and call process()

                self.entry_obsolete = 0

            # Take care of keywords like
            # msgid, msgid_plural, msgctxt & msgstr.
            if tokens[0] in keywords and nb_tokens > 1:
                line = line[len(tokens[0]) :].lstrip()
                if re.search(r'([^\\]|^)"', line[1:-1]):
                    raise IOError(
                        'Syntax error in po file %s (line %s): '
                        'unescaped double quote found'
                        % (self.instance.fpath, self.current_line)
                    )
                self.current_token = line
                self.process(keywords[tokens[0]])
                continue

            self.current_token = line

                self.process('oc')

            elif line[:1] == '"':
                # we are on a continuation line
                if re.search(r'([^\\]|^)"', line[1:-1]):
                    raise IOError(
                        'Syntax error in po file %s (line %s): '
                        'unescaped double quote found'
                        % (self.instance.fpath, self.current_line)
                    )
                self.process('mc')

            elif line[:7] == 'msgstr[':
                # we are on a msgstr plural
                self.process('mx')

                # we are on a generated comment line
                self.process('gc')

            elif tokens[0] == '#|':
                if nb_tokens <= 1:
                    raise IOError(
                        'Syntax error in po file %s (line %s)'
                        % (self.instance.fpath, self.current_line)
                    )

                # Remove the marker and any whitespace right after that.
                line = line[2:].lstrip()
                self.current_token = line

                    self.process('mc')
                    continue

                if nb_tokens == 2:
                    # Invalid continuation line.
                    raise IOError(
                        'Syntax error in po file %s (line %s): '
                        'invalid continuation line'
                        % (self.instance.fpath, self.current_line)
                    )

                # we are on a "previous translation" comment line,
                if tokens[1] not in prev_keywords:
                    # Unknown keyword in previous translation comment.
                    raise IOError(
                        'Syntax error in po file %s (line %s): '
                        'unknown keyword %s'
                        % (self.instance.fpath, self.current_line, tokens[1])
                    )

                # Remove the keyword and any whitespace
                # between it and the starting quote.
                line = line[len(tokens[1]) :].lstrip()
                self.current_token = line
                self.process(prev_keywords[tokens[1]])

            else:
                raise IOError(
                    'Syntax error in po file %s (line %s)'
                    % (self.instance.fpath, self.current_line)
                )

        if (
            self.current_entry
            and len(tokens) > 0
            and not tokens[0].startswith('#')
        ):
            # since entries are added when another entry is found, we must add
            # the last entry here (only if there are lines). Trailing comments
            # are ignored
            self.instance.append(self.current_entry)

        try:
            (action, state) = self.transitions[(symbol, self.current_state)]
            if action():
                self.current_state = state
        except Exception:
            raise IOError(
                'Syntax error in po file (line %s)' % self.current_line
            )

    # state handlers

    def handle_he(self):
        """Handle a header comment."""

    def handle_fl(self):
        """Handle a flags line."""
        if self.current_state in ['mc', 'ms', 'mx']:
            self.instance.append(self.current_entry)
            self.current_entry = POEntry(linenum=self.current_line)
        self.current_entry.flags += [
            c.strip() for c in self.current_token[3:].split(',')
        ]
        return True

    def handle_pp(self):
        """Handle a previous msgid_plural line."""
        if self.current_state in ['mc', 'ms', 'mx']:
            self.instance.append(self.current_entry)
            self.current_entry = POEntry(linenum=self.current_line)
        self.current_entry.previous_msgid_plural = unescape(
            self.current_token[1:-1]
        )
        return True

    def handle_pm(self):
        """Handle a previous msgid line."""
        if self.current_state in ['mc', 'ms', 'mx']:
            self.instance.append(self.current_entry)
            self.current_entry = POEntry(linenum=self.current_line)
        self.current_entry.previous_msgid = unescape(self.current_token[1:-1])
        return True

    def handle_pc(self):
        """Handle a previous msgctxt line."""
        if self.current_state in ['mc', 'ms', 'mx']:
            self.instance.append(self.current_entry)
            self.current_entry = POEntry(linenum=self.current_line)
        self.current_entry.previous_msgctxt = unescape(self.current_token[1:-1])
        return True

    def handle_ct(self):
        """Handle a msgctxt."""
        if self.current_state in ['mc', 'ms', 'mx']:

        return True

    def handle_mx(self):
        """Handle a msgstr plural."""
        index = self.current_token[7]
        value = self.current_token[self.current_token.find('"') + 1 : -1]
        self.current_entry.msgstr_plural[int(index)] = unescape(value)
        self.msgstr_index = int(index)
        return True

    def handle_mc(self):

            self.current_entry.previous_msgid += token
        elif self.current_state == 'pc':
            self.current_entry.previous_msgctxt += token
        # don't change the current state
        return False


# }}}
# class _MOFileParser {{{


class _MOFileParser(object):

        if klass is None:
            klass = MOFile
        self.instance = klass(
            fpath=mofile,
            encoding=kwargs.get('encoding', default_encoding),
            check_for_duplicates=kwargs.get('check_for_duplicates', False),
        )

    def __del__(self):
        """
        Make sure the file is closed, this prevents warnings on unclosed file

            msgid_tokens = msgid.split(b('\0'))
            if len(msgid_tokens) > 1:
                entry = self._build_entry(
                    msgid=msgid_tokens[0],
                    msgid_plural=msgid_tokens[1],
                    msgstr_plural=dict(
                        (k, v) for k, v in enumerate(msgstr.split(b('\0')))
                    ),
                )
            else:
                entry = self._build_entry(msgid=msgid, msgstr=msgstr)
            self.instance.append(entry)
        # close opened file
        self.fhandle.close()
        return self.instance

    def _build_entry(
        self, msgid, msgstr=None, msgid_plural=None, msgstr_plural=None
    ):
        msgctxt_msgid = msgid.split(b('\x04'))
        encoding = self.instance.encoding
        if len(msgctxt_msgid) > 1:
            kwargs = {
                'msgctxt': msgctxt_msgid[0].decode(encoding),

        bytes = self.fhandle.read(numbytes)
        tup = struct.unpack(fmt, bytes)
        if len(tup) == 1:
            return tup[0]
        return tup


# }}}
# class TextWrapper {{{


class TextWrapper(textwrap.TextWrapper):
    """
    Subclass of textwrap.TextWrapper that backport the
    drop_whitespace option.
    """

    def __init__(self, *args, **kwargs):
        drop_whitespace = kwargs.pop('drop_whitespace', True)
        textwrap.TextWrapper.__init__(self, *args, **kwargs)
        self.drop_whitespace = drop_whitespace

            # of all lines (return value).
            if cur_line:
                lines.append(indent + ''.join(cur_line))

        return lines


# }}}
# function wrap() {{{


def wrap(text, width=70, **kwargs):

    """
    if sys.version_info < (2, 6):
        return TextWrapper(width=width, **kwargs).wrap(text)
    return textwrap.wrap(text, width=width, **kwargs)


# }}}