From 5cf010475586dd0a294659e6de57c2424157d345 Mon Sep 17 00:00:00 2001 From: PwndDepot Date: Fri, 23 Apr 2021 01:55:09 -0600 Subject: [PATCH 1/3] MasterClass Add new extractor --- youtube_dl/extractor/extractors.py | 1 + youtube_dl/extractor/masterclass.py | 77 +++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+) create mode 100644 youtube_dl/extractor/masterclass.py diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py index ac33cd996..82a126630 100644 --- a/youtube_dl/extractor/extractors.py +++ b/youtube_dl/extractor/extractors.py @@ -640,6 +640,7 @@ from .mangomolo import ( MangomoloLiveIE, ) from .manyvids import ManyVidsIE +from .masterclass import MasterClassIE from .maoritv import MaoriTVIE from .markiza import ( MarkizaIE, diff --git a/youtube_dl/extractor/masterclass.py b/youtube_dl/extractor/masterclass.py new file mode 100644 index 000000000..cde6d3088 --- /dev/null +++ b/youtube_dl/extractor/masterclass.py @@ -0,0 +1,77 @@ +# coding: utf-8 +from __future__ import unicode_literals +import re + +from .brightcove import BrightcoveNewIE +from .common import InfoExtractor + + +class MasterClassIE(InfoExtractor): + IE_NAME = 'masterclass' + _VALID_URL = r'https:\/\/www.masterclass.com[\/]classes[\/](.+?)(?=[\/]|$)(?:[\/]chapters)?(?:[\/])?(.+?(?=[?]|$))?' + _TESTS = [{ + 'url': 'https://www.masterclass.com/classes/gordon-ramsay-teaches-cooking-restaurant-recipes-at-home/chapters/introduction-to-gordon-ramsay', + 'md5': '5fbe8736e761c2cc61ed0ee2841e8f47', + 'info_dict': { + 'id': '5834174464001', + 'ext': 'mp4', + 'title': 'GR2_Trailer_v9a_Brightcove_v2', + 'timestamp': 1536792485, + 'upload_date': '20180912', + 'uploader_id': '5344802162001', + }, + 'skip': 'Requires masterclass account credentials', + }] + + BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/%s/default_default/index.html?videoId=%s' + BRIGHTCOVE_ACCOUNT_ID = '5344802162001' + + def _real_extract(self, url): + course_id, video_id = re.match(self._VALID_URL, url).groups() + + headers = {"Connection": "keep-alive", + "Cache-Control": "max-age=0", + "Upgrade-Insecure-Requests": "1"} + webpage = self._download_webpage(url, course_id, headers=headers) + + match_free = re.search(r'\"embedUrl\":\"(.+?)\"}', webpage) + + if match_free: # match free video + clip_url = match_free.groups()[0] + return self.url_result(clip_url) + + csrf = self._html_search_regex(r'name=\"csrf-token\" content=\"(.+?)(?=\" \/)', webpage, 'csrf') + headers = {"Content-Type": "application/vnd.api+json", + "X-CSRF-Token": csrf, + "Accept": "application/vnd.api+json"} + + json_data = self._download_json('https://www.masterclass.com/jsonapi/v1/courses/%s?include=instructors%%2Cchapters%%2Cchapters.annotations%%2Cchapters.subchapters%%2Cpdfs' % course_id, course_id, headers=headers) + + entries = [] + for json_obj in json_data.get('included'): + if (video_id and json_obj.get('attributes').get('slug') == video_id) or (not video_id and json_obj.get('type') == 'chapter'): + brightcove_id = json_obj.get('attributes').get('brightcove_video_id') + brightcove_url = self.BRIGHTCOVE_URL_TEMPLATE % (self.BRIGHTCOVE_ACCOUNT_ID, brightcove_id) + + title = json_obj.get('attributes').get('title') + playlist = json_data.get('data').get('attributes').get('title') + playlist_index = json_obj.get('attributes').get('number') + + entries.append({ + '_type': 'url_transparent', + 'url': brightcove_url, + 'ie_key': BrightcoveNewIE.ie_key(), + 'id': brightcove_id, + 'title': title, + 'playlist': playlist, + 'playlist_index': playlist_index + }) + + if video_id and len(entries) == 1: + return entries[0] + elif not video_id: + entries = sorted(entries, key=lambda i: int(i['playlist_index'])) + title = json_data.get('data').get('attributes').get('title') + slug = json_data.get('data').get('attributes').get('slug') + description = json_data.get('data').get('attributes').get('headline') + return self.playlist_result(entries, slug, title, description) From 8f745151ee932c6c68905d645e1a535bbf131380 Mon Sep 17 00:00:00 2001 From: PwndDepot Date: Thu, 29 Apr 2021 17:13:16 -0500 Subject: [PATCH 2/3] Updated regex and extractor code and separated course extraction --- youtube_dl/extractor/extractors.py | 5 +- youtube_dl/extractor/masterclass.py | 134 ++++++++++++++++++++-------- 2 files changed, 101 insertions(+), 38 deletions(-) diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py index 82a126630..3c7cf96c9 100644 --- a/youtube_dl/extractor/extractors.py +++ b/youtube_dl/extractor/extractors.py @@ -640,13 +640,16 @@ from .mangomolo import ( MangomoloLiveIE, ) from .manyvids import ManyVidsIE -from .masterclass import MasterClassIE from .maoritv import MaoriTVIE from .markiza import ( MarkizaIE, MarkizaPageIE, ) from .massengeschmacktv import MassengeschmackTVIE +from .masterclass import ( + MasterClassIE, + MasterClassCourseIE, +) from .matchtv import MatchTVIE from .mdr import MDRIE from .medaltv import MedalTVIE diff --git a/youtube_dl/extractor/masterclass.py b/youtube_dl/extractor/masterclass.py index cde6d3088..2e9f9b8c6 100644 --- a/youtube_dl/extractor/masterclass.py +++ b/youtube_dl/extractor/masterclass.py @@ -4,13 +4,32 @@ import re from .brightcove import BrightcoveNewIE from .common import InfoExtractor +from ..utils import ( + int_or_none, + url_or_none, + parse_duration, + unified_strdate +) -class MasterClassIE(InfoExtractor): +class MasterClassBaseIE(InfoExtractor): + BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/%s/default_default/index.html?videoId=%s' + BRIGHTCOVE_ACCOUNT_ID = '5344802162001' + + def _call_api(self, course_id, csrf, video_id): + headers = {"Content-Type": "application/vnd.api+json", + "X-CSRF-Token": csrf, + "Accept": "application/vnd.api+json"} + + return self._download_json('https://www.masterclass.com/jsonapi/v1/courses/%s?include=instructors%%2Cchapters.annotations' % course_id, + video_id, headers=headers, fatal=True) + + +class MasterClassIE(MasterClassBaseIE): IE_NAME = 'masterclass' - _VALID_URL = r'https:\/\/www.masterclass.com[\/]classes[\/](.+?)(?=[\/]|$)(?:[\/]chapters)?(?:[\/])?(.+?(?=[?]|$))?' + _VALID_URL = r'https?://(?:www\.)?masterclass\.com/classes/(?P[a-z0-9\-]+)/chapters/(?P[a-z0-9\-]+)' _TESTS = [{ - 'url': 'https://www.masterclass.com/classes/gordon-ramsay-teaches-cooking-restaurant-recipes-at-home/chapters/introduction-to-gordon-ramsay', + 'url': 'https://www.masterclass.com/classes/gordon-ramsay-teaches-cooking-restaurant-recipes-at-home/chapters/pomme-puree', 'md5': '5fbe8736e761c2cc61ed0ee2841e8f47', 'info_dict': { 'id': '5834174464001', @@ -19,59 +38,100 @@ class MasterClassIE(InfoExtractor): 'timestamp': 1536792485, 'upload_date': '20180912', 'uploader_id': '5344802162001', - }, - 'skip': 'Requires masterclass account credentials', + 'duration': 93.44 + } }] - BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/%s/default_default/index.html?videoId=%s' - BRIGHTCOVE_ACCOUNT_ID = '5344802162001' - def _real_extract(self, url): course_id, video_id = re.match(self._VALID_URL, url).groups() - - headers = {"Connection": "keep-alive", - "Cache-Control": "max-age=0", - "Upgrade-Insecure-Requests": "1"} - webpage = self._download_webpage(url, course_id, headers=headers) + webpage = self._download_webpage(url, video_id) match_free = re.search(r'\"embedUrl\":\"(.+?)\"}', webpage) - - if match_free: # match free video + if match_free: # match free trailer video clip_url = match_free.groups()[0] return self.url_result(clip_url) - csrf = self._html_search_regex(r'name=\"csrf-token\" content=\"(.+?)(?=\" \/)', webpage, 'csrf') - headers = {"Content-Type": "application/vnd.api+json", - "X-CSRF-Token": csrf, - "Accept": "application/vnd.api+json"} + csrf = self._html_search_regex(r'name=\"csrf-token\" content=\"(.+?)(?=\" \/)', webpage, 'csrf', fatal=True) + meta = self._call_api(course_id, csrf, video_id) + + chapter = meta.get('data').get('attributes').get('title') + release_date = unified_strdate(meta.get('data').get('attributes').get('available_at')) + for json_data in meta['included']: + json_data = json_data['attributes'] + if (json_data['slug'] == video_id): + brightcove_id = json_data['brightcove_video_id'] + brightcove_url = self.BRIGHTCOVE_URL_TEMPLATE % (self.BRIGHTCOVE_ACCOUNT_ID, brightcove_id) + + title = json_data['title'] + chapter_number = json_data.get('number') + duration = int_or_none(parse_duration(json_data.get('duration'))) + description = json_data.get('abstract') + thumbnail = url_or_none(json_data.get('video_thumb_url')) + + return ({ + '_type': 'url_transparent', + 'url': brightcove_url, + 'ie_key': BrightcoveNewIE.ie_key(), + 'id': video_id, + 'title': title, + 'chapter': chapter, + 'chapter_number': chapter_number, + 'release_date': release_date, + 'duration': duration, + 'description': description, + 'thumbnail': thumbnail + }) + + +class MasterClassCourseIE(MasterClassBaseIE): + IE_NAME = "masterclass:course" + _VALID_URL = r'https?://(?:www\.)?masterclass\.com/classes/(?P[^/]+)' + _TESTS = [{ + 'url': 'https://www.masterclass.com/classes/alicia-keys-teaches-songwriting-and-producing', + 'only_matching': True, + }, { + 'url': 'https://www.masterclass.com/classes/matthew-walker-teaches-the-science-of-better-sleep', + 'only_matching': True, + }] + + def _real_extract(self, url): + course_id = self._match_id(url) + webpage = self._download_webpage(url, course_id, fatal=True) - json_data = self._download_json('https://www.masterclass.com/jsonapi/v1/courses/%s?include=instructors%%2Cchapters%%2Cchapters.annotations%%2Cchapters.subchapters%%2Cpdfs' % course_id, course_id, headers=headers) + csrf = self._html_search_regex(r'name=\"csrf-token\" content=\"(.+?)(?=\" \/)', webpage, 'csrf', fatal=True) + meta = self._call_api(course_id, csrf, course_id) + chapter = meta.get('data').get('attributes').get('title') + release_date = unified_strdate(meta.get('data').get('attributes').get('available_at')) entries = [] - for json_obj in json_data.get('included'): - if (video_id and json_obj.get('attributes').get('slug') == video_id) or (not video_id and json_obj.get('type') == 'chapter'): - brightcove_id = json_obj.get('attributes').get('brightcove_video_id') + for json_data in meta['included']: + if (json_data['type'] == 'chapter'): + json_data = json_data['attributes'] + brightcove_id = json_data['brightcove_video_id'] brightcove_url = self.BRIGHTCOVE_URL_TEMPLATE % (self.BRIGHTCOVE_ACCOUNT_ID, brightcove_id) - title = json_obj.get('attributes').get('title') - playlist = json_data.get('data').get('attributes').get('title') - playlist_index = json_obj.get('attributes').get('number') + title = json_data['title'] + video_id = json_data.get('slug') + chapter_number = json_data.get('number') + duration = int_or_none(parse_duration(json_data.get('duration'))) + description = json_data.get('abstract') + thumbnail = url_or_none(json_data.get('video_thumb_url')) entries.append({ '_type': 'url_transparent', 'url': brightcove_url, 'ie_key': BrightcoveNewIE.ie_key(), - 'id': brightcove_id, + 'id': video_id, 'title': title, - 'playlist': playlist, - 'playlist_index': playlist_index + 'chapter': chapter, + 'chapter_number': chapter_number, + 'release_date': release_date, + 'duration': duration, + 'description': description, + 'thumbnail': thumbnail }) - if video_id and len(entries) == 1: - return entries[0] - elif not video_id: - entries = sorted(entries, key=lambda i: int(i['playlist_index'])) - title = json_data.get('data').get('attributes').get('title') - slug = json_data.get('data').get('attributes').get('slug') - description = json_data.get('data').get('attributes').get('headline') - return self.playlist_result(entries, slug, title, description) + entries = sorted(entries, key=lambda i: int(i.get("chapter_number"))) + title = meta['data']['attributes']['title'] + description = meta.get('data').get('attributes').get('overview') + return self.playlist_result(entries, course_id, title, description) From 206796d6b37d0af9e8e3699a30d3b1301b7e2c06 Mon Sep 17 00:00:00 2001 From: PwnDepot Date: Thu, 6 May 2021 17:44:00 -0600 Subject: [PATCH 3/3] Added int_or_none to chapter number --- youtube_dl/extractor/masterclass.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/masterclass.py b/youtube_dl/extractor/masterclass.py index 2e9f9b8c6..8ab0d56c8 100644 --- a/youtube_dl/extractor/masterclass.py +++ b/youtube_dl/extractor/masterclass.py @@ -63,7 +63,7 @@ class MasterClassIE(MasterClassBaseIE): brightcove_url = self.BRIGHTCOVE_URL_TEMPLATE % (self.BRIGHTCOVE_ACCOUNT_ID, brightcove_id) title = json_data['title'] - chapter_number = json_data.get('number') + chapter_number = int_or_none(json_data.get('number')) duration = int_or_none(parse_duration(json_data.get('duration'))) description = json_data.get('abstract') thumbnail = url_or_none(json_data.get('video_thumb_url')) @@ -112,7 +112,7 @@ class MasterClassCourseIE(MasterClassBaseIE): title = json_data['title'] video_id = json_data.get('slug') - chapter_number = json_data.get('number') + chapter_number = int_or_none(json_data.get('number')) duration = int_or_none(parse_duration(json_data.get('duration'))) description = json_data.get('abstract') thumbnail = url_or_none(json_data.get('video_thumb_url'))