From 051e014e5606be7aeedea905090ddca55b00f000 Mon Sep 17 00:00:00 2001 From: df Date: Sat, 11 Sep 2021 11:47:17 +0100 Subject: [PATCH 01/13] Calculate file hash on an initial portion, if specified External downloaders may not support limiting the download to a specified size --- test/test_download.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test/test_download.py b/test/test_download.py index ebe820dfc..329f569c2 100644 --- a/test/test_download.py +++ b/test/test_download.py @@ -37,6 +37,7 @@ from youtube_dl.utils import ( UnavailableVideoError, ) from youtube_dl.extractor import get_info_extractor +from youtube_dl.downloader.common import FileDownloader RETRIES = 3 @@ -56,9 +57,10 @@ class YoutubeDL(youtube_dl.YoutubeDL): return super(YoutubeDL, self).process_info(info_dict) -def _file_md5(fn): +def _file_md5(fn, length=None): with open(fn, 'rb') as f: - return hashlib.md5(f.read()).hexdigest() + return hashlib.md5( + f.read(length) if length is not None else f.read()).hexdigest() defs = gettestcases() @@ -224,7 +226,7 @@ def generator(test_case, tname): (tc_filename, format_bytes(expected_minsize), format_bytes(got_fsize))) if 'md5' in tc: - md5_for_file = _file_md5(tc_filename) + md5_for_file = _file_md5(tc_filename) if not params.get('test') else _file_md5(tc_filename, FileDownloader._TEST_FILE_SIZE) self.assertEqual(tc['md5'], md5_for_file) # Finally, check test cases' data again but this time against # extracted data from info JSON file written during processing From 0c8f7a5f124500c618e75a6ce0cf1c27eaa2e546 Mon Sep 17 00:00:00 2001 From: df Date: Sat, 11 Sep 2021 11:58:05 +0100 Subject: [PATCH 02/13] Support header-related params in test cases Header-related params are parsed into `std_headers` rather than the `params` dict, but this wasn't respected for test cases. --- test/test_download.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test/test_download.py b/test/test_download.py index 329f569c2..c4e6c8697 100644 --- a/test/test_download.py +++ b/test/test_download.py @@ -34,6 +34,7 @@ from youtube_dl.utils import ( DownloadError, ExtractorError, format_bytes, + std_headers, UnavailableVideoError, ) from youtube_dl.extractor import get_info_extractor @@ -125,6 +126,17 @@ def generator(test_case, tname): params.setdefault('extract_flat', 'in_playlist') params.setdefault('skip_download', True) + if 'user_agent' in params: + std_headers['User-Agent'] = params['user_agent'] + + if 'referer' in params: + std_headers['Referer'] = params['referer'] + + for h in params.get('headers', []): + h = h.split(':', 1) + if len(h) > 1: + std_headers[h[0]] = h[1] + ydl = YoutubeDL(params, auto_init=False) ydl.add_default_info_extractors() finished_hook_called = set() From 5ab2dbeced73252395396ae0cb9b5ced1649074c Mon Sep 17 00:00:00 2001 From: df Date: Sat, 11 Sep 2021 12:13:09 +0100 Subject: [PATCH 03/13] Add a TestSuite named 'test_%(ie_key)s_all' for each extractor So TestDownload.test_XXX_all runs all the download test cases for the XXX extractor. --- test/test_download.py | 57 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 51 insertions(+), 6 deletions(-) diff --git a/test/test_download.py b/test/test_download.py index c4e6c8697..468901ca4 100644 --- a/test/test_download.py +++ b/test/test_download.py @@ -23,6 +23,7 @@ import hashlib import io import json import socket +import re import youtube_dl.YoutubeDL from youtube_dl.compat import ( @@ -86,6 +87,13 @@ class TestDownload(unittest.TestCase): strclass(self.__class__), ' [%s]' % add_ie if add_ie else '') + @classmethod + def addTest(cls, test_method, test_method_name, add_ie): + test_method.__name__ = str(test_method_name) + test_method.add_ie = add_ie + setattr(TestDownload, test_method.__name__, test_method) + del test_method + def setUp(self): self.defs = defs @@ -268,12 +276,49 @@ for n, test_case in enumerate(defs): tname = 'test_%s_%d' % (test_case['name'], i) i += 1 test_method = generator(test_case, tname) - test_method.__name__ = str(tname) - ie_list = test_case.get('add_ie') - test_method.add_ie = ie_list and ','.join(ie_list) - setattr(TestDownload, test_method.__name__, test_method) - del test_method - + ie_list = ','.join(test_case.get('add_ie', [])) + TestDownload.addTest(test_method, tname, ie_list) + +# Py2 compat (should be in compat.py?) +try: + from itertools import (ifilter as filter, imap as map) +except ImportError: + pass + + +def tests_for_ie(ie_key): + return filter( + lambda a: callable(getattr(TestDownload, a, None)), + filter(lambda a: re.match(r'test_%s(?:_\d+)?$' % ie_key, a), + dir(TestDownload))) + + +def gen_test_suite(ie_key): + def test_all(self): + print(self) + suite = unittest.TestSuite( + map(TestDownload, tests_for_ie(ie_key))) + result = self.defaultTestResult() + suite.run(result) + print('Errors: %d\t Failures: %d\tSkipped: %d' % + tuple(map(len, (result.errors, result.failures, result.skipped)))) + print('Expected failures: %d\tUnexpected successes: %d' % + tuple(map(len, (result.expectedFailures, result.unexpectedSuccesses)))) + return result + + return test_all + + +for ie_key in set( + map(lambda x: x[0], + filter( + lambda x: callable(x[1]), + map(lambda a: (a[5:], getattr(TestDownload, a, None)), + filter(lambda t: + re.match(r'test_.+(? Date: Sat, 28 Jan 2023 18:15:41 +0000 Subject: [PATCH 04/13] Condense filters Co-authored-by: Simon Sawicki --- test/test_download.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/test/test_download.py b/test/test_download.py index 468901ca4..16797134e 100644 --- a/test/test_download.py +++ b/test/test_download.py @@ -310,13 +310,12 @@ def gen_test_suite(ie_key): for ie_key in set( - map(lambda x: x[0], + map(lambda a: a[5:], filter( - lambda x: callable(x[1]), - map(lambda a: (a[5:], getattr(TestDownload, a, None)), - filter(lambda t: - re.match(r'test_.+(? Date: Sat, 28 Jan 2023 18:21:33 +0000 Subject: [PATCH 05/13] Use compat_[filter,map] --- test/test_download.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/test/test_download.py b/test/test_download.py index 16797134e..6dfa5f263 100644 --- a/test/test_download.py +++ b/test/test_download.py @@ -27,6 +27,8 @@ import re import youtube_dl.YoutubeDL from youtube_dl.compat import ( + compat_filter as filter, + compat_map as map, compat_http_client, compat_urllib_error, compat_HTTPError, @@ -279,12 +281,6 @@ for n, test_case in enumerate(defs): ie_list = ','.join(test_case.get('add_ie', [])) TestDownload.addTest(test_method, tname, ie_list) -# Py2 compat (should be in compat.py?) -try: - from itertools import (ifilter as filter, imap as map) -except ImportError: - pass - def tests_for_ie(ie_key): return filter( From 40b4b691ee25f66920ef781a3b2631efcc1b2025 Mon Sep 17 00:00:00 2001 From: dirkf Date: Wed, 21 Feb 2024 11:18:37 +0000 Subject: [PATCH 06/13] Apply suggestions from code review --- test/test_download.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_download.py b/test/test_download.py index e0cf7d1e2..eb8cfa1df 100644 --- a/test/test_download.py +++ b/test/test_download.py @@ -65,7 +65,7 @@ class YoutubeDL(youtube_dl.YoutubeDL): def _file_md5(fn, length=None): with open(fn, 'rb') as f: return hashlib.md5( - f.read(length) if length is not None else f.read()).hexdigest() + f.read() if length is None else f.read(length)).hexdigest() defs = gettestcases() @@ -94,7 +94,7 @@ class TestDownload(unittest.TestCase): def addTest(cls, test_method, test_method_name, add_ie): test_method.__name__ = str(test_method_name) test_method.add_ie = add_ie - setattr(TestDownload, test_method.__name__, test_method) + setattr(cls, test_method.__name__, test_method) del test_method def setUp(self): From 6ae54a47a7bcca38c625a9acecf6deeb5dc1dad0 Mon Sep 17 00:00:00 2001 From: dirkf Date: Wed, 21 Feb 2024 11:25:26 +0000 Subject: [PATCH 07/13] Update test/test_download.py --- test/test_download.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_download.py b/test/test_download.py index eb8cfa1df..9b5a5702f 100644 --- a/test/test_download.py +++ b/test/test_download.py @@ -28,8 +28,8 @@ import re import youtube_dl.YoutubeDL from youtube_dl.compat import ( compat_filter as filter, - compat_map as map, compat_http_client, + compat_map as map, compat_urllib_error, compat_HTTPError, ) From a9f55cb69be40d22f6fe6a32eafc8febdfe3d4f2 Mon Sep 17 00:00:00 2001 From: dirkf Date: Wed, 21 Feb 2024 11:29:34 +0000 Subject: [PATCH 08/13] Update test/test_download.py --- test/test_download.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/test_download.py b/test/test_download.py index 6cf4577ab..92f8b7bf9 100644 --- a/test/test_download.py +++ b/test/test_download.py @@ -29,7 +29,6 @@ from youtube_dl.compat import ( compat_filter as filter, compat_http_client, compat_map as map, - compat_urllib_error, compat_HTTPError, compat_open as open, compat_urllib_error, From f7173cca4d6cf09ac7e6d82495783a57a5003442 Mon Sep 17 00:00:00 2001 From: dirkf Date: Wed, 21 Feb 2024 12:49:59 +0000 Subject: [PATCH 09/13] Update test_download.py --- test/test_download.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_download.py b/test/test_download.py index 92f8b7bf9..5dbf123a7 100644 --- a/test/test_download.py +++ b/test/test_download.py @@ -28,8 +28,8 @@ import youtube_dl.YoutubeDL from youtube_dl.compat import ( compat_filter as filter, compat_http_client, - compat_map as map, compat_HTTPError, + compat_map as map, compat_open as open, compat_urllib_error, ) From 2bd001fa8c85870610f20c69220fabc519370c0f Mon Sep 17 00:00:00 2001 From: dirkf Date: Wed, 21 Feb 2024 13:07:25 +0000 Subject: [PATCH 10/13] Retry loop with itertools --- test/test_download.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/test/test_download.py b/test/test_download.py index 5dbf123a7..9b07837d5 100644 --- a/test/test_download.py +++ b/test/test_download.py @@ -20,6 +20,7 @@ from test.helper import ( import hashlib +import itertools import json import socket import re @@ -175,8 +176,7 @@ def generator(test_case, tname): try_rm_tcs_files() try: - try_num = 1 - while True: + for try_num in itertools.count(1): try: # We're not using .download here since that is just a shim # for outside error handling, and returns the exit code @@ -185,7 +185,7 @@ def generator(test_case, tname): test_case['url'], force_generic_extractor=params.get('force_generic_extractor', False)) except (DownloadError, ExtractorError) as err: - # Check if the exception is not a network related one + # Retry, or raise if the exception is not network-related if not err.exc_info[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError, compat_http_client.BadStatusLine) or (err.exc_info[0] == compat_HTTPError and err.exc_info[1].code == 503): msg = getattr(err, 'msg', error_to_compat_str(err)) err.msg = '%s (%s)' % (msg, tname, ) @@ -196,8 +196,6 @@ def generator(test_case, tname): return print('Retrying: {0} failed tries\n\n##########\n\n'.format(try_num)) - - try_num += 1 else: break From 15b06163a8378d7cf42881883b54e9854081052d Mon Sep 17 00:00:00 2001 From: dirkf Date: Wed, 21 Feb 2024 15:29:16 +0000 Subject: [PATCH 11/13] Make signature of fake YoutubeDL.report_warning() match the real one (per 640d39f) --- test/test_download.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_download.py b/test/test_download.py index 9b07837d5..4e728e980 100644 --- a/test/test_download.py +++ b/test/test_download.py @@ -54,7 +54,7 @@ class YoutubeDL(youtube_dl.YoutubeDL): self.processed_info_dicts = [] super(YoutubeDL, self).__init__(*args, **kwargs) - def report_warning(self, message): + def report_warning(self, message, *args, **kwargs): # Don't accept warnings during tests raise ExtractorError(message) From 2239666542e431760903d45f2b799b954d813988 Mon Sep 17 00:00:00 2001 From: dirkf Date: Wed, 21 Feb 2024 15:44:04 +0000 Subject: [PATCH 12/13] Handle `expected_warnings` better * make fake `report_warning()` method signatures correct (per 640d39f) * support single warning to expect as well as sequence * don't colour text to be matched * use `expected_warnings()` function throughout --- test/helper.py | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/test/helper.py b/test/helper.py index 5b7e3dfe2..527ce3a94 100644 --- a/test/helper.py +++ b/test/helper.py @@ -20,6 +20,7 @@ from youtube_dl.compat import ( from youtube_dl.utils import ( IDENTITY, preferredencoding, + variadic, write_string, ) @@ -66,7 +67,7 @@ def report_warning(message): class FakeYDL(YoutubeDL): def __init__(self, override=None): # Different instances of the downloader can't share the same dictionary - # some test set the "sublang" parameter, which would break the md5 checks. + # some tests set the "sublang" parameter, which would break the md5 checks. params = get_params(override=override) super(FakeYDL, self).__init__(params, auto_init=False) self.result = [] @@ -83,14 +84,7 @@ class FakeYDL(YoutubeDL): def expect_warning(self, regex): # Silence an expected warning matching a regex - old_report_warning = self.report_warning - - def report_warning(self, message): - if re.match(regex, message): - return - old_report_warning(message) - self.report_warning = types.MethodType(report_warning, self) - + expect_warnings(self, regex) class FakeLogger(object): def debug(self, msg): @@ -285,12 +279,14 @@ def assertEqual(self, got, expected, msg=None): def expect_warnings(ydl, warnings_re): real_warning = ydl.report_warning + # to facilitate matching, don't prettify messages + ydl.params['no_color'] = True - def _report_warning(w): - if not any(re.search(w_re, w) for w_re in warnings_re): - real_warning(w) + def _report_warning(self, w, *args, **kwargs): + if not any(re.search(w_re, w) for w_re in variadic(warnings_re)): + real_warning(w, *args, **kwargs) - ydl.report_warning = _report_warning + ydl.report_warning = types.MethodType(_report_warning, ydl) def http_server_port(httpd): From fdb694eca1da55584450db3ab7ff6a5f49ecc29f Mon Sep 17 00:00:00 2001 From: dirkf Date: Wed, 21 Feb 2024 15:48:06 +0000 Subject: [PATCH 13/13] Update helper.py - linted --- test/helper.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/helper.py b/test/helper.py index 527ce3a94..d51ed5634 100644 --- a/test/helper.py +++ b/test/helper.py @@ -86,6 +86,7 @@ class FakeYDL(YoutubeDL): # Silence an expected warning matching a regex expect_warnings(self, regex) + class FakeLogger(object): def debug(self, msg): pass @@ -279,7 +280,7 @@ def assertEqual(self, got, expected, msg=None): def expect_warnings(ydl, warnings_re): real_warning = ydl.report_warning - # to facilitate matching, don't prettify messages + # to facilitate matching, don't prettify messages ydl.params['no_color'] = True def _report_warning(self, w, *args, **kwargs):