pagure/tests/test_pagure_lib_encoding_utils.py

# -*- coding: utf-8 -*-
"""
Tests for :module:`pagure.lib.encoding_utils`.
"""

from __future__ import unicode_literals, absolute_import

import chardet
import os
import unittest
import sys

sys.path.insert(0, os.path.join(os.path.dirname(
    os.path.abspath(__file__)), '..'))

from pagure.lib import encoding_utils


class TestGuessEncoding(unittest.TestCase):

    def test_guess_encoding_ascii(self):
        """
        Assert when ascii-only data is provided ascii is the guessed encoding.
        """
        data = 'Twas bryllyg, and the slythy toves did gyre and gymble'
        result = encoding_utils.guess_encoding(data.encode('ascii'))
        self.assertEqual(result, 'ascii')

    def test_guess_encoding_favor_utf_8(self):
        """
        Test that strings that could be UTF-8 or ISO-8859-* result in UTF-8.

        python-chardet-3.0.4-2.fc27.noarch detects it as ISO-8859-9
        python-chardet-2.2.1-1.el7_1.noarch detects it as ISO-8859-2
        """
        data = 'Šabata'.encode('utf-8')
        result = encoding_utils.guess_encoding(data)
        chardet_result = chardet.detect(data)
        self.assertEqual(result, 'utf-8')
        if chardet.__version__[0] == '3':
            self.assertEqual(chardet_result['encoding'], 'ISO-8859-9')
        else:
            self.assertEqual(chardet_result['encoding'], 'ISO-8859-2')

    def test_guess_encoding_no_data(self):
        """ Test encoding_utils.guess_encoding() with an empty string """
        result = encoding_utils.guess_encoding(''.encode('utf-8'))
        self.assertEqual(result, 'ascii')


class TestGuessEncodings(unittest.TestCase):

    def test_guess_encodings(self):
        """ Test the encoding_utils.guess_encodings() method. """
        data = 'Šabata'.encode('utf-8')
        result = encoding_utils.guess_encodings(data)
        chardet_result = chardet.detect(data)
        if chardet.__version__[0] == '3':
            # The first three have different confidence values
            self.assertListEqual(
                [encoding.encoding for encoding in result][:3],
                ['utf-8', 'ISO-8859-9', 'ISO-8859-1']
            )
            # This is the one with the least confidence
            self.assertEqual(result[-1].encoding, 'windows-1255')
            # The values in the middle of the list all have the same confidence
            # value and can't be sorted reliably: use sets.
            self.assertEqual(
                set([encoding.encoding for encoding in result]),
                set(['utf-8', 'ISO-8859-9', 'ISO-8859-1', 'MacCyrillic',
                     'IBM866', 'TIS-620', 'EUC-JP', 'EUC-KR', 'GB2312',
                     'KOI8-R', 'Big5', 'IBM855', 'ISO-8859-7', 'SHIFT_JIS',
                     'windows-1253', 'CP949', 'EUC-TW', 'ISO-8859-5',
                     'windows-1251', 'windows-1255'])
            )
            self.assertEqual(chardet_result['encoding'], 'ISO-8859-9')
        else:
            self.assertListEqual(
                [encoding.encoding for encoding in result],
                ['utf-8', 'ISO-8859-2', 'windows-1252'])
            self.assertEqual(chardet_result['encoding'], 'ISO-8859-2')

    def test_guess_encodings_no_data(self):
        """ Test encoding_utils.guess_encodings() with an emtpy string """
        result = encoding_utils.guess_encodings(''.encode('utf-8'))
        self.assertEqual(
            [encoding.encoding for encoding in result],
            ['ascii'])

class TestDecode(unittest.TestCase):

    def test_decode(self):
        """ Test encoding_utils.decode() """
        data = 'Šabata'
        self.assertEqual(data, encoding_utils.decode(data.encode('utf-8')))


if __name__ == '__main__':
    unittest.main(verbosity=2)