gallery-dl/test/test_util.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-

# Copyright 2015-2022 Mike Fährmann
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.

import os
import sys
import unittest

import io
import random
import string
import datetime
import itertools
import http.cookiejar

sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from gallery_dl import util, text, exception  # noqa E402


class TestRange(unittest.TestCase):

    def test_parse_range(self, f=util.RangePredicate.parse_range):
        self.assertEqual(
            f(""),
            [])
        self.assertEqual(
            f("1-2"),
            [(1, 2)])
        self.assertEqual(
            f("-"),
            [(1, sys.maxsize)])
        self.assertEqual(
            f("-2,4,6-8,10-"),
            [(1, 2), (4, 4), (6, 8), (10, sys.maxsize)])
        self.assertEqual(
            f(" - 3 , 4-  4, 2-6"),
            [(1, 3), (4, 4), (2, 6)])

    def test_optimize_range(self, f=util.RangePredicate.optimize_range):
        self.assertEqual(
            f([]),
            [])
        self.assertEqual(
            f([(2, 4)]),
            [(2, 4)])
        self.assertEqual(
            f([(2, 4), (6, 8), (10, 12)]),
            [(2, 4), (6, 8), (10, 12)])
        self.assertEqual(
            f([(2, 4), (4, 6), (5, 8)]),
            [(2, 8)])
        self.assertEqual(
            f([(1, 1), (2, 2), (3, 6), (8, 9)]),
            [(1, 6), (8, 9)])


class TestPredicate(unittest.TestCase):

    def test_range_predicate(self):
        dummy = None

        pred = util.RangePredicate(" - 3 , 4-  4, 2-6")
        for i in range(6):
            self.assertTrue(pred(dummy, dummy))
        with self.assertRaises(exception.StopExtraction):
            bool(pred(dummy, dummy))

        pred = util.RangePredicate("1, 3, 5")
        self.assertTrue(pred(dummy, dummy))
        self.assertFalse(pred(dummy, dummy))
        self.assertTrue(pred(dummy, dummy))
        self.assertFalse(pred(dummy, dummy))
        self.assertTrue(pred(dummy, dummy))
        with self.assertRaises(exception.StopExtraction):
            bool(pred(dummy, dummy))

        pred = util.RangePredicate("")
        with self.assertRaises(exception.StopExtraction):
            bool(pred(dummy, dummy))

    def test_unique_predicate(self):
        dummy = None
        pred = util.UniquePredicate()

        # no duplicates
        self.assertTrue(pred("1", dummy))
        self.assertTrue(pred("2", dummy))
        self.assertFalse(pred("1", dummy))
        self.assertFalse(pred("2", dummy))
        self.assertTrue(pred("3", dummy))
        self.assertFalse(pred("3", dummy))

        # duplicates for "text:"
        self.assertTrue(pred("text:123", dummy))
        self.assertTrue(pred("text:123", dummy))
        self.assertTrue(pred("text:123", dummy))

    def test_filter_predicate(self):
        url = ""

        pred = util.FilterPredicate("a < 3")
        self.assertTrue(pred(url, {"a": 2}))
        self.assertFalse(pred(url, {"a": 3}))

        with self.assertRaises(SyntaxError):
            util.FilterPredicate("(")

        with self.assertRaises(exception.FilterError):
            util.FilterPredicate("a > 1")(url, {"a": None})

        with self.assertRaises(exception.FilterError):
            util.FilterPredicate("b > 1")(url, {"a": 2})

        pred = util.FilterPredicate(["a < 3", "b < 4", "c < 5"])
        self.assertTrue(pred(url, {"a": 2, "b": 3, "c": 4}))
        self.assertFalse(pred(url, {"a": 3, "b": 3, "c": 4}))
        self.assertFalse(pred(url, {"a": 2, "b": 4, "c": 4}))
        self.assertFalse(pred(url, {"a": 2, "b": 3, "c": 5}))
        with self.assertRaises(exception.FilterError):
            pred(url, {"a": 2})

    def test_build_predicate(self):
        pred = util.build_predicate([])
        self.assertIsInstance(pred, type(lambda: True))

        pred = util.build_predicate([util.UniquePredicate()])
        self.assertIsInstance(pred, util.UniquePredicate)

        pred = util.build_predicate([util.UniquePredicate(),
                                     util.UniquePredicate()])
        self.assertIs(pred.func, util.chain_predicates)


class TestISO639_1(unittest.TestCase):

    def test_code_to_language(self):
        d = "default"
        self._run_test(util.code_to_language, {
            ("en",): "English",
            ("FR",): "French",
            ("ja",): "Japanese",
            ("xx",): None,
            (""  ,): None,
            (None,): None,
            ("en", d): "English",
            ("FR", d): "French",
            ("xx", d): d,
            (""  , d): d,
            (None, d): d,
        })

    def test_language_to_code(self):
        d = "default"
        self._run_test(util.language_to_code, {
            ("English",): "en",
            ("fRENch",): "fr",
            ("Japanese",): "ja",
            ("xx",): None,
            (""  ,): None,
            (None,): None,
            ("English", d): "en",
            ("fRENch", d): "fr",
            ("xx", d): d,
            (""  , d): d,
            (None, d): d,
        })

    def _run_test(self, func, tests):
        for args, result in tests.items():
            self.assertEqual(func(*args), result)


class TestCookiesTxt(unittest.TestCase):

    def test_cookiestxt_load(self):

        def _assert(content, expected):
            jar = http.cookiejar.CookieJar()
            util.cookiestxt_load(io.StringIO(content, None), jar)
            for c, e in zip(jar, expected):
                self.assertEqual(c.__dict__, e.__dict__)

        _assert("", [])
        _assert("\n\n\n", [])
        _assert("$ Comment", [])
        _assert("# Comment", [])
        _assert(" # Comment \n\n $ Comment ", [])
        _assert(
            ".example.org\tTRUE\t/\tTRUE\t0\tname\tvalue",
            [self._cookie("name", "value", ".example.org")],
        )
        _assert(
            ".example.org\tTRUE\t/\tTRUE\t\tname\t",
            [self._cookie("name", "", ".example.org")],
        )
        _assert(
            "\tTRUE\t/\tTRUE\t\tname\t",
            [self._cookie("name", "", "")],
        )
        _assert(
            "# Netscape HTTP Cookie File\n"
            "\n"
            "# default\n"
            ".example.org	TRUE	/	FALSE	0	n1	v1\n"
            ".example.org	TRUE	/	TRUE	2145945600	n2	v2\n"
            ".example.org	TRUE	/path	FALSE	0		n3\n"
            "\n"
            "  # # extra # #  \n"
            "www.example.org	FALSE	/	FALSE		n4	\n"
            "www.example.org	FALSE	/path	FALSE	100	n5	v5\n",
            [
                self._cookie(
                    "n1", "v1", ".example.org", True, "/", False),
                self._cookie(
                    "n2", "v2", ".example.org", True, "/", True, 2145945600),
                self._cookie(
                    "n3", None, ".example.org", True, "/path", False),
                self._cookie(
                    "n4", ""  , "www.example.org", False, "/", False),
                self._cookie(
                    "n5", "v5", "www.example.org", False, "/path", False, 100),
            ],
        )

        with self.assertRaises(ValueError):
            util.cookiestxt_load("example.org\tTRUE\t/\tTRUE\t0\tname",
                                 http.cookiejar.CookieJar())

    def test_cookiestxt_store(self):

        def _assert(cookies, expected):
            fp = io.StringIO(newline=None)
            util.cookiestxt_store(fp, cookies)
            self.assertMultiLineEqual(fp.getvalue(), expected)

        _assert([], "# Netscape HTTP Cookie File\n\n")
        _assert(
            [self._cookie("name", "value", ".example.org")],
            "# Netscape HTTP Cookie File\n\n"
            ".example.org\tTRUE\t/\tTRUE\t0\tname\tvalue\n",
        )
        _assert(
            [
                self._cookie(
                    "n1", "v1", ".example.org", True, "/", False),
                self._cookie(
                    "n2", "v2", ".example.org", True, "/", True, 2145945600),
                self._cookie(
                    "n3", None, ".example.org", True, "/path", False),
                self._cookie(
                    "n4", ""  , "www.example.org", False, "/", False),
                self._cookie(
                    "n5", "v5", "www.example.org", False, "/path", False, 100),
                self._cookie(
                    "n6", "v6", "", False),
            ],
            "# Netscape HTTP Cookie File\n"
            "\n"
            ".example.org	TRUE	/	FALSE	0	n1	v1\n"
            ".example.org	TRUE	/	TRUE	2145945600	n2	v2\n"
            ".example.org	TRUE	/path	FALSE	0		n3\n"
            "www.example.org	FALSE	/	FALSE	0	n4	\n"
            "www.example.org	FALSE	/path	FALSE	100	n5	v5\n",
        )

    def _cookie(self, name, value, domain, domain_specified=True,
                path="/", secure=True, expires=None):
        return http.cookiejar.Cookie(
            0, name, value, None, False,
            domain, domain_specified, domain.startswith("."),
            path, False, secure, expires, False, None, None, {},
        )


class TestOther(unittest.TestCase):

    def test_bencode(self):
        self.assertEqual(util.bencode(0), "")
        self.assertEqual(util.bencode(123), "123")
        self.assertEqual(util.bencode(123, "01"), "1111011")
        self.assertEqual(util.bencode(123, "BA"), "AAAABAA")

    def test_bdecode(self):
        self.assertEqual(util.bdecode(""), 0)
        self.assertEqual(util.bdecode("123"), 123)
        self.assertEqual(util.bdecode("1111011", "01"), 123)
        self.assertEqual(util.bdecode("AAAABAA", "BA"), 123)

    def test_bencode_bdecode(self):
        for _ in range(100):
            value = random.randint(0, 1000000)
            for alphabet in ("01", "0123456789", string.ascii_letters):
                result = util.bdecode(util.bencode(value, alphabet), alphabet)
                self.assertEqual(result, value)

    def test_advance(self):
        items = range(5)

        self.assertCountEqual(
            util.advance(items, 0), items)
        self.assertCountEqual(
            util.advance(items, 3), range(3, 5))
        self.assertCountEqual(
            util.advance(items, 9), [])
        self.assertCountEqual(
            util.advance(util.advance(items, 1), 2), range(3, 5))

    def test_unique(self):
        self.assertSequenceEqual(
            list(util.unique("")), "")
        self.assertSequenceEqual(
            list(util.unique("AABBCC")), "ABC")
        self.assertSequenceEqual(
            list(util.unique("ABABABCAABBCC")), "ABC")
        self.assertSequenceEqual(
            list(util.unique([1, 2, 1, 3, 2, 1])), [1, 2, 3])

    def test_unique_sequence(self):
        self.assertSequenceEqual(
            list(util.unique_sequence("")), "")
        self.assertSequenceEqual(
            list(util.unique_sequence("AABBCC")), "ABC")
        self.assertSequenceEqual(
            list(util.unique_sequence("ABABABCAABBCC")), "ABABABCABC")
        self.assertSequenceEqual(
            list(util.unique_sequence([1, 2, 1, 3, 2, 1])), [1, 2, 1, 3, 2, 1])

    def test_contains(self):
        c = [1, "2", 3, 4, "5", "foo"]
        self.assertTrue(util.contains(c, 1))
        self.assertTrue(util.contains(c, "foo"))
        self.assertTrue(util.contains(c, [1, 3, "5"]))
        self.assertTrue(util.contains(c, ["a", "b", "5"]))
        self.assertFalse(util.contains(c, "bar"))
        self.assertFalse(util.contains(c, [2, 5, "bar"]))

        s = "1 2 3 asd qwe y(+)c f(+)(-) bar"
        self.assertTrue(util.contains(s, "y(+)c"))
        self.assertTrue(util.contains(s, ["asd", "qwe", "yxc"]))
        self.assertTrue(util.contains(s, ["sdf", "dfg", "qwe"]))
        self.assertFalse(util.contains(s, "tag1"))
        self.assertFalse(util.contains(s, ["tag1", "tag2", "tag3"]))

        s = "1, 2, 3, asd, qwe, y(+)c, f(+)(-), bar"
        self.assertTrue(util.contains(s, "y(+)c", ", "))
        self.assertTrue(util.contains(s, ["sdf", "dfg", "qwe"], ", "))
        self.assertFalse(util.contains(s, "tag1", ", "))

    def test_raises(self):
        func = util.raises(Exception)
        with self.assertRaises(Exception):
            func()

        func = util.raises(ValueError)
        with self.assertRaises(ValueError):
            func(1)
        with self.assertRaises(ValueError):
            func(2)
        with self.assertRaises(ValueError):
            func(3)

    def test_identity(self):
        for value in (123, "foo", [1, 2, 3], (1, 2, 3), {1: 2}, None):
            self.assertIs(util.identity(value), value)

    def test_noop(self):
        self.assertEqual(util.noop(), None)

    def test_compile_expression(self):
        expr = util.compile_expression("1 + 2 * 3")
        self.assertEqual(expr(), 7)
        self.assertEqual(expr({"a": 1, "b": 2, "c": 3}), 7)
        self.assertEqual(expr({"a": 9, "b": 9, "c": 9}), 7)

        expr = util.compile_expression("a + b * c")
        self.assertEqual(expr({"a": 1, "b": 2, "c": 3}), 7)
        self.assertEqual(expr({"a": 9, "b": 9, "c": 9}), 90)

        with self.assertRaises(NameError):
            expr()
        with self.assertRaises(NameError):
            expr({"a": 2})

        with self.assertRaises(SyntaxError):
            util.compile_expression("")
        with self.assertRaises(SyntaxError):
            util.compile_expression("x++")

        expr = util.compile_expression("1 and abort()")
        with self.assertRaises(exception.StopExtraction):
            expr()

    def test_build_duration_func(self, f=util.build_duration_func):

        def test_single(df, v):
            for _ in range(10):
                self.assertEqual(df(), v)

        def test_range(df, lower, upper):
            for __ in range(10):
                v = df()
                self.assertGreaterEqual(v, lower)
                self.assertLessEqual(v, upper)

        for v in (0, 0.0, "", None, (), []):
            self.assertIsNone(f(v))

        for v in (0, 0.0, "", None, (), []):
            test_single(f(v, 1.0), 1.0)

        test_single(f(3), 3)
        test_single(f(3.0), 3.0)
        test_single(f("3"), 3)
        test_single(f("3.0-"), 3)
        test_single(f("  3  -"), 3)

        test_range(f((2, 4)), 2, 4)
        test_range(f([2, 4]), 2, 4)
        test_range(f("2-4"), 2, 4)
        test_range(f("  2.0  - 4 "), 2, 4)

    def test_extractor_filter(self):
        # empty
        func = util.build_extractor_filter("")
        self.assertEqual(func(TestExtractor)      , True)
        self.assertEqual(func(TestExtractorParent), True)
        self.assertEqual(func(TestExtractorAlt)   , True)

        # category
        func = util.build_extractor_filter("test_category")
        self.assertEqual(func(TestExtractor)      , False)
        self.assertEqual(func(TestExtractorParent), False)
        self.assertEqual(func(TestExtractorAlt)   , True)

        # subcategory
        func = util.build_extractor_filter("*:test_subcategory")
        self.assertEqual(func(TestExtractor)      , False)
        self.assertEqual(func(TestExtractorParent), True)
        self.assertEqual(func(TestExtractorAlt)   , False)

        # basecategory
        func = util.build_extractor_filter("test_basecategory")
        self.assertEqual(func(TestExtractor)      , False)
        self.assertEqual(func(TestExtractorParent), False)
        self.assertEqual(func(TestExtractorAlt)   , False)

        # category-subcategory pair
        func = util.build_extractor_filter("test_category:test_subcategory")
        self.assertEqual(func(TestExtractor)      , False)
        self.assertEqual(func(TestExtractorParent), True)
        self.assertEqual(func(TestExtractorAlt)   , True)

        # combination
        func = util.build_extractor_filter(
            ["test_category", "*:test_subcategory"])
        self.assertEqual(func(TestExtractor)      , False)
        self.assertEqual(func(TestExtractorParent), False)
        self.assertEqual(func(TestExtractorAlt)   , False)

        # whitelist
        func = util.build_extractor_filter(
            "test_category:test_subcategory", negate=False)
        self.assertEqual(func(TestExtractor)      , True)
        self.assertEqual(func(TestExtractorParent), False)
        self.assertEqual(func(TestExtractorAlt)   , False)

        func = util.build_extractor_filter(
            ["test_category:test_subcategory", "*:test_subcategory_parent"],
            negate=False)
        self.assertEqual(func(TestExtractor)      , True)
        self.assertEqual(func(TestExtractorParent), True)
        self.assertEqual(func(TestExtractorAlt)   , False)

    def test_generate_token(self):
        tokens = set()
        for _ in range(100):
            token = util.generate_token()
            tokens.add(token)
            self.assertEqual(len(token), 16 * 2)
            self.assertRegex(token, r"^[0-9a-f]+$")
        self.assertGreaterEqual(len(tokens), 99)

        token = util.generate_token(80)
        self.assertEqual(len(token), 80 * 2)
        self.assertRegex(token, r"^[0-9a-f]+$")

    def test_format_value(self):
        self.assertEqual(util.format_value(0)         , "0")
        self.assertEqual(util.format_value(1)         , "1")
        self.assertEqual(util.format_value(12)        , "12")
        self.assertEqual(util.format_value(123)       , "123")
        self.assertEqual(util.format_value(1234)      , "1.23k")
        self.assertEqual(util.format_value(12345)     , "12.34k")
        self.assertEqual(util.format_value(123456)    , "123.45k")
        self.assertEqual(util.format_value(1234567)   , "1.23M")
        self.assertEqual(util.format_value(12345678)  , "12.34M")
        self.assertEqual(util.format_value(123456789) , "123.45M")
        self.assertEqual(util.format_value(1234567890), "1.23G")

    def test_combine_dict(self):
        self.assertEqual(
            util.combine_dict({}, {}),
            {})
        self.assertEqual(
            util.combine_dict({1: 1, 2: 2}, {2: 4, 4: 8}),
            {1: 1, 2: 4, 4: 8})
        self.assertEqual(
            util.combine_dict(
                {1: {11: 22, 12: 24}, 2: {13: 26, 14: 28}},
                {1: {11: 33, 13: 39}, 2: "str"}),
            {1: {11: 33, 12: 24, 13: 39}, 2: "str"})
        self.assertEqual(
            util.combine_dict(
                {1: {2: {3: {4: {"1": "a", "2": "b"}}}}},
                {1: {2: {3: {4: {"1": "A", "3": "C"}}}}}),
            {1: {2: {3: {4: {"1": "A", "2": "b", "3": "C"}}}}})

    def test_transform_dict(self):
        d = {}
        util.transform_dict(d, str)
        self.assertEqual(d, {})

        d = {1: 123, 2: "123", 3: True, 4: None}
        util.transform_dict(d, str)
        self.assertEqual(
            d, {1: "123", 2: "123", 3: "True", 4: "None"})

        d = {1: 123, 2: "123", 3: "foo", 4: {11: 321, 12: "321", 13: "bar"}}
        util.transform_dict(d, text.parse_int)
        self.assertEqual(
            d, {1: 123, 2: 123, 3: 0, 4: {11: 321, 12: 321, 13: 0}})

    def test_filter_dict(self):
        d = {}
        r = util.filter_dict(d)
        self.assertEqual(r, d)
        self.assertIsNot(r, d)

        d = {"foo": 123, "bar": [], "baz": None}
        r = util.filter_dict(d)
        self.assertEqual(r, d)
        self.assertIsNot(r, d)

        d = {"foo": 123, "_bar": [], "__baz__": None}
        r = util.filter_dict(d)
        self.assertEqual(r, {"foo": 123})

    def test_enumerate_reversed(self):

        seq = [11, 22, 33]
        result = [(3, 33), (2, 22), (1, 11)]

        def gen():
            for i in seq:
                yield i

        def gen_2():
            yield from seq

        def assertEqual(it1, it2):
            ae = self.assertEqual
            for i1, i2 in itertools.zip_longest(it1, it2):
                ae(i1, i2)

        assertEqual(
            util.enumerate_reversed(seq), [(2, 33), (1, 22), (0, 11)])
        assertEqual(
            util.enumerate_reversed(seq, 1), result)
        assertEqual(
            util.enumerate_reversed(seq, 2), [(4, 33), (3, 22), (2, 11)])

        assertEqual(
            util.enumerate_reversed(gen(), 0, len(seq)),
            [(2, 33), (1, 22), (0, 11)])
        assertEqual(
            util.enumerate_reversed(gen(), 1, len(seq)), result)
        assertEqual(
            util.enumerate_reversed(gen_2(), 1, len(seq)), result)
        assertEqual(
            util.enumerate_reversed(gen_2(), 2, len(seq)),
            [(4, 33), (3, 22), (2, 11)])

    def test_number_to_string(self, f=util.number_to_string):
        self.assertEqual(f(1)     , "1")
        self.assertEqual(f(1.0)   , "1.0")
        self.assertEqual(f("1.0") , "1.0")
        self.assertEqual(f([1])   , [1])
        self.assertEqual(f({1: 2}), {1: 2})
        self.assertEqual(f(True)  , True)
        self.assertEqual(f(None)  , None)

    def test_to_string(self, f=util.to_string):
        self.assertEqual(f(1)    , "1")
        self.assertEqual(f(1.0)  , "1.0")
        self.assertEqual(f("1.0"), "1.0")

        self.assertEqual(f("")   , "")
        self.assertEqual(f(None) , "")
        self.assertEqual(f(0)    , "")

        self.assertEqual(f(["a"]), "a")
        self.assertEqual(f([1])  , "1")
        self.assertEqual(f(["a", "b", "c"]), "a, b, c")
        self.assertEqual(f([1, 2, 3]), "1, 2, 3")

    def test_datetime_to_timestamp(self, f=util.datetime_to_timestamp):
        self.assertEqual(f(util.EPOCH), 0.0)
        self.assertEqual(f(datetime.datetime(2010, 1, 1)), 1262304000.0)
        self.assertEqual(f(datetime.datetime(2010, 1, 1, 0, 0, 0, 128000)),
                         1262304000.128000)
        with self.assertRaises(TypeError):
            f(None)

    def test_datetime_to_timestamp_string(
            self, f=util.datetime_to_timestamp_string):
        self.assertEqual(f(util.EPOCH), "0")
        self.assertEqual(f(datetime.datetime(2010, 1, 1)), "1262304000")
        self.assertEqual(f(None), "")

    def test_universal_none(self):
        obj = util.NONE

        self.assertFalse(obj)
        self.assertEqual(len(obj), 0)
        self.assertEqual(str(obj), str(None))
        self.assertEqual(repr(obj), repr(None))
        self.assertEqual(format(obj), str(None))
        self.assertEqual(format(obj, "%F"), str(None))
        self.assertIs(obj.attr, obj)
        self.assertIs(obj["key"], obj)
        self.assertIs(obj(), obj)
        self.assertIs(obj(1, "a"), obj)
        self.assertIs(obj(foo="bar"), obj)

        i = 0
        for _ in obj:
            i += 1
        self.assertEqual(i, 0)


class TestExtractor():
    category = "test_category"
    subcategory = "test_subcategory"
    basecategory = "test_basecategory"


class TestExtractorParent(TestExtractor):
    category = "test_category"
    subcategory = "test_subcategory_parent"


class TestExtractorAlt(TestExtractor):
    category = "test_category_alt"
    subcategory = "test_subcategory"


if __name__ == '__main__':
    unittest.main()