From b478a0ac3f57f587ca3302469feaec8a0f034f9b Mon Sep 17 00:00:00 2001 From: 01001110 Date: Thu, 23 Mar 2023 20:00:44 +0800 Subject: [PATCH 1/7] [chelseafc] Add new extractor --- youtube_dl/extractor/chelseafc.py | 73 ++++++++++++++++++++++++++++++ youtube_dl/extractor/extractors.py | 1 + 2 files changed, 74 insertions(+) create mode 100644 youtube_dl/extractor/chelseafc.py diff --git a/youtube_dl/extractor/chelseafc.py b/youtube_dl/extractor/chelseafc.py new file mode 100644 index 000000000..5402ca801 --- /dev/null +++ b/youtube_dl/extractor/chelseafc.py @@ -0,0 +1,73 @@ +# coding: utf-8 +from __future__ import unicode_literals + +import json + +from .common import InfoExtractor +from ..utils import unified_timestamp, parse_duration + + +class ChelseafcIE(InfoExtractor): + _VALID_URL = r'https?://(?:www\.)?chelseafc\.com(?:/[a-z]+)?/video/(?P[a-z0-9]+(?:-[a-z0-9]+)*)' + _TESTS = [{ + 'url': 'https://www.chelseafc.com/en/video/full-match-chelsea-2-2-everton', + 'md5': '0f7cd37de17faa71566bfd9074315c85', + 'info_dict': { + 'id': 'full-match-chelsea-2-2-everton', + 'ext': 'mp4', + 'title': 'Full Match: Chelsea 2-2 Everton', + 'description': 'Full match highlights from Chelsea\'s 2-2 Premier League draw with Everton at Stamford Bridge. ', + 'duration': 2842.0, + 'timestamp': 1679184000, + 'upload_date': '20230319', + 'thumbnail': r're:https?://.*\.png' + } + }, + { + 'url': 'https://www.chelseafc.com/en/video/manchester-city-vs-chelsea-2-0-or-highlights-or-efl-cup', + 'md5': '01078658408ee98b1cf286d3b8f4b7ca', + 'info_dict': { + 'id': 'manchester-city-vs-chelsea-2-0-or-highlights-or-efl-cup', + 'ext': 'mp4', + 'title': 'Manchester City 2-0 Chelsea | Highlights | EFL Cup', + 'description': 'Highlights from our EFL Cup match against Man City.', + 'duration': 120.0, + 'timestamp': 1668042000, + 'upload_date': '20221110', + 'thumbnail': r're:https?://.*\.jpg' + } + }] + + def _real_extract(self, url): + video_id = self._match_id(url) + + webpage = self._download_webpage(url, video_id) + + raw_data = self._html_search_regex( + # TODO improve regex + r'(?:]+(?:data-component="VideoDetails".*?)+data-props="([^"]*))', + webpage, + 'data' + ) + data = json.loads(raw_data)['videoDetail'] + + manifest_url = data['signedUrl'] + formats = self._extract_m3u8_formats(manifest_url, video_id, 'mp4') + + title = data['title'] + descripiton = data['description'] + timestamp = unified_timestamp(data['releaseDate']) + duration = parse_duration(data['duration']) + tags = [tag['title'] for tag in data['tags']] + thumbnail = data['image']['file']['url'] + + return { + 'id': video_id, + 'title': title, + 'description': descripiton, + 'formats': formats, + 'duration': duration, + 'timestamp': timestamp, + 'tags': tags, + 'thumbnail': thumbnail + } diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py index 3a87f9e33..2a4620049 100644 --- a/youtube_dl/extractor/extractors.py +++ b/youtube_dl/extractor/extractors.py @@ -1682,3 +1682,4 @@ from .pr0gramm import ( Pr0grammIE, Pr0grammStaticIE, ) +from .chelseafc import ChelseafcIE From ceaa2c78fa17c3d1f89f7cc6481ce7d2bf4512fe Mon Sep 17 00:00:00 2001 From: 01001110 Date: Fri, 24 Mar 2023 16:22:13 +0800 Subject: [PATCH 2/7] [chelseafc] improve regex --- youtube_dl/extractor/chelseafc.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/youtube_dl/extractor/chelseafc.py b/youtube_dl/extractor/chelseafc.py index 5402ca801..5c3234243 100644 --- a/youtube_dl/extractor/chelseafc.py +++ b/youtube_dl/extractor/chelseafc.py @@ -43,12 +43,17 @@ class ChelseafcIE(InfoExtractor): webpage = self._download_webpage(url, video_id) - raw_data = self._html_search_regex( - # TODO improve regex - r'(?:]+(?:data-component="VideoDetails".*?)+data-props="([^"]*))', + video_details_div = self._search_regex( + r'(]*\sdata-component\s*=\s*(?:"|\')\s*VideoDetails\s*(?:"|\')[^>]*>)', webpage, + 'div' + ) + raw_data = self._html_search_regex( + r']*\sdata-props\s*=\s*(?:"|\')\s*([^"\']*)\s*(?:"|\')[^>]*>', + video_details_div, 'data' ) + data = json.loads(raw_data)['videoDetail'] manifest_url = data['signedUrl'] From a9a4d683647f2e979c21fd6b72244e232545a683 Mon Sep 17 00:00:00 2001 From: 01001110 Date: Sun, 2 Apr 2023 18:28:44 +0800 Subject: [PATCH 3/7] Edit tests --- youtube_dl/extractor/chelseafc.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/youtube_dl/extractor/chelseafc.py b/youtube_dl/extractor/chelseafc.py index 5c3234243..5d4776c7b 100644 --- a/youtube_dl/extractor/chelseafc.py +++ b/youtube_dl/extractor/chelseafc.py @@ -11,21 +11,22 @@ class ChelseafcIE(InfoExtractor): _VALID_URL = r'https?://(?:www\.)?chelseafc\.com(?:/[a-z]+)?/video/(?P[a-z0-9]+(?:-[a-z0-9]+)*)' _TESTS = [{ 'url': 'https://www.chelseafc.com/en/video/full-match-chelsea-2-2-everton', - 'md5': '0f7cd37de17faa71566bfd9074315c85', + 'md5': '16c0df4e6909ad5ae96eab68fc584d75', 'info_dict': { 'id': 'full-match-chelsea-2-2-everton', 'ext': 'mp4', 'title': 'Full Match: Chelsea 2-2 Everton', - 'description': 'Full match highlights from Chelsea\'s 2-2 Premier League draw with Everton at Stamford Bridge. ', + 'description': 'Full match highlights from Chelsea\'s 2-2 Premier League draw with Everton at Stamford Bridge.', 'duration': 2842.0, 'timestamp': 1679184000, + 'tags': ['Premier League', 'Everton', 'Video and article choice'], 'upload_date': '20230319', 'thumbnail': r're:https?://.*\.png' } }, { 'url': 'https://www.chelseafc.com/en/video/manchester-city-vs-chelsea-2-0-or-highlights-or-efl-cup', - 'md5': '01078658408ee98b1cf286d3b8f4b7ca', + 'md5': 'c20c52f3954d938bc5da4d6fe358c81d', 'info_dict': { 'id': 'manchester-city-vs-chelsea-2-0-or-highlights-or-efl-cup', 'ext': 'mp4', @@ -34,6 +35,7 @@ class ChelseafcIE(InfoExtractor): 'duration': 120.0, 'timestamp': 1668042000, 'upload_date': '20221110', + 'tags': ['Highlights', 'League Cup', 'Manchester City'], 'thumbnail': r're:https?://.*\.jpg' } }] From 6a9848f94a1a2c9f5886cfa5de9425e6c532a964 Mon Sep 17 00:00:00 2001 From: 01001110 Date: Sun, 2 Apr 2023 18:32:21 +0800 Subject: [PATCH 4/7] Modify extractor according to suggestions --- youtube_dl/extractor/chelseafc.py | 35 +++++++++++++++++-------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/youtube_dl/extractor/chelseafc.py b/youtube_dl/extractor/chelseafc.py index 5d4776c7b..b999354b5 100644 --- a/youtube_dl/extractor/chelseafc.py +++ b/youtube_dl/extractor/chelseafc.py @@ -1,10 +1,13 @@ # coding: utf-8 from __future__ import unicode_literals -import json - from .common import InfoExtractor -from ..utils import unified_timestamp, parse_duration +from ..utils import ( + parse_duration, + traverse_obj, + unified_timestamp, + url_or_none, +) class ChelseafcIE(InfoExtractor): @@ -56,25 +59,25 @@ class ChelseafcIE(InfoExtractor): 'data' ) - data = json.loads(raw_data)['videoDetail'] + data = self._parse_json(raw_data, video_id) + manifest_url = data['videoDetail']['signedUrl'] - manifest_url = data['signedUrl'] - formats = self._extract_m3u8_formats(manifest_url, video_id, 'mp4') + data = data['videoDetail'] title = data['title'] - descripiton = data['description'] - timestamp = unified_timestamp(data['releaseDate']) - duration = parse_duration(data['duration']) - tags = [tag['title'] for tag in data['tags']] - thumbnail = data['image']['file']['url'] + + formats = self._extract_m3u8_formats(manifest_url, video_id, 'mp4') + self._sort_formats(formats) + + txt_or_none = lambda x: x.strip() or None return { 'id': video_id, 'title': title, - 'description': descripiton, + 'description': txt_or_none(data.get('description')), 'formats': formats, - 'duration': duration, - 'timestamp': timestamp, - 'tags': tags, - 'thumbnail': thumbnail + 'duration': parse_duration(data.get('duration')), + 'timestamp': unified_timestamp(data.get('releaseDate')), + 'tags': traverse_obj(data, ('tags', Ellipsis, 'title'), expected_type=txt_or_none), + 'thumbnail': traverse_obj(data, ('image', 'file', 'url'), expected_type=url_or_none), } From 24e84f000084e5ba50547cac6c2ed1a0ada861f2 Mon Sep 17 00:00:00 2001 From: 01001110 Date: Sun, 2 Apr 2023 19:31:05 +0800 Subject: [PATCH 5/7] Change regex for better matching --- youtube_dl/extractor/chelseafc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/chelseafc.py b/youtube_dl/extractor/chelseafc.py index b999354b5..ad10251c8 100644 --- a/youtube_dl/extractor/chelseafc.py +++ b/youtube_dl/extractor/chelseafc.py @@ -49,7 +49,7 @@ class ChelseafcIE(InfoExtractor): webpage = self._download_webpage(url, video_id) video_details_div = self._search_regex( - r'(]*\sdata-component\s*=\s*(?:"|\')\s*VideoDetails\s*(?:"|\')[^>]*>)', + r'(]*\bdata-component\s*=\s*(?:"|\')\s*VideoDetails\s*(?:"|\')[^>]*>)', webpage, 'div' ) From 705925382023b3dc1253d7461e0a1e74553cec13 Mon Sep 17 00:00:00 2001 From: dirkf Date: Thu, 6 Apr 2023 01:45:06 +0100 Subject: [PATCH 6/7] Update extractors.py * sort extractor modules --- youtube_dl/extractor/extractors.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py index 2a4620049..432bbc69d 100644 --- a/youtube_dl/extractor/extractors.py +++ b/youtube_dl/extractor/extractors.py @@ -214,6 +214,7 @@ from .ceskatelevize import CeskaTelevizeIE from .channel9 import Channel9IE from .charlierose import CharlieRoseIE from .chaturbate import ChaturbateIE +from .chelseafc import ChelseafcIE from .chilloutzone import ChilloutzoneIE from .chirbit import ( ChirbitIE, @@ -979,6 +980,10 @@ from .puhutv import ( PuhuTVIE, PuhuTVSerieIE, ) +from .pr0gramm import ( + Pr0grammIE, + Pr0grammStaticIE, +) from .presstv import PressTVIE from .prosiebensat1 import ProSiebenSat1IE from .puls4 import Puls4IE @@ -1678,8 +1683,3 @@ from .zingmp3 import ( ) from .zoom import ZoomIE from .zype import ZypeIE -from .pr0gramm import ( - Pr0grammIE, - Pr0grammStaticIE, -) -from .chelseafc import ChelseafcIE From b8f7389a712f0962a17927b8ce9557d50fb6a3ca Mon Sep 17 00:00:00 2001 From: 01001110 Date: Tue, 3 Oct 2023 00:29:54 +0800 Subject: [PATCH 7/7] update tests checksums --- youtube_dl/extractor/chelseafc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/chelseafc.py b/youtube_dl/extractor/chelseafc.py index ad10251c8..05f885412 100644 --- a/youtube_dl/extractor/chelseafc.py +++ b/youtube_dl/extractor/chelseafc.py @@ -14,7 +14,7 @@ class ChelseafcIE(InfoExtractor): _VALID_URL = r'https?://(?:www\.)?chelseafc\.com(?:/[a-z]+)?/video/(?P[a-z0-9]+(?:-[a-z0-9]+)*)' _TESTS = [{ 'url': 'https://www.chelseafc.com/en/video/full-match-chelsea-2-2-everton', - 'md5': '16c0df4e6909ad5ae96eab68fc584d75', + 'md5': '2fda617911b7148a2a19bec55b75d30a', 'info_dict': { 'id': 'full-match-chelsea-2-2-everton', 'ext': 'mp4', @@ -29,7 +29,7 @@ class ChelseafcIE(InfoExtractor): }, { 'url': 'https://www.chelseafc.com/en/video/manchester-city-vs-chelsea-2-0-or-highlights-or-efl-cup', - 'md5': 'c20c52f3954d938bc5da4d6fe358c81d', + 'md5': '2905365c3c9cf4612f303fbb99c2f4ca', 'info_dict': { 'id': 'manchester-city-vs-chelsea-2-0-or-highlights-or-efl-cup', 'ext': 'mp4',