From 082dfa495e6e4e7a3a8da36b93e04b7c7a4525f2 Mon Sep 17 00:00:00 2001 From: Fynn Date: Mon, 6 Sep 2021 14:10:01 +0200 Subject: [PATCH 1/2] [snapchat] Add new extractor --- youtube_dl/extractor/extractors.py | 1 + youtube_dl/extractor/snapchat.py | 64 ++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+) create mode 100644 youtube_dl/extractor/snapchat.py diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py index 6e8fc3961..58ce207d3 100644 --- a/youtube_dl/extractor/extractors.py +++ b/youtube_dl/extractor/extractors.py @@ -1109,6 +1109,7 @@ from .sky import ( from .slideshare import SlideshareIE from .slideslive import SlidesLiveIE from .slutload import SlutloadIE +from .snapchat import SnapchatIE from .snotr import SnotrIE from .sohu import SohuIE from .sonyliv import SonyLIVIE diff --git a/youtube_dl/extractor/snapchat.py b/youtube_dl/extractor/snapchat.py new file mode 100644 index 000000000..37389d5d6 --- /dev/null +++ b/youtube_dl/extractor/snapchat.py @@ -0,0 +1,64 @@ +# coding: utf-8 +from __future__ import unicode_literals +import re + +from ..utils import unified_timestamp, parse_duration +from .common import InfoExtractor + + +class SnapchatIE(InfoExtractor): + _VALID_URL = r'https?://(?:www\.)?story.snapchat\.com/o/(?P\w+)' + _TEST = { + 'url': 'https://story.snapchat.com/o/W7_EDlXWTBiXAEEniNoMPwAAYuz9_1mcdex8MAXndPyIOAXndPyFyAO1OAA', + 'md5': 'ab1900981cadcd955aae32a526096cbd', + 'info_dict': { + 'id': 'W7_EDlXWTBiXAEEniNoMPwAAYuz9_1mcdex8MAXndPyIOAXndPyFyAO1OAA', + 'ext': 'mp4', + 'title': 'W7_EDlXWTBiXAEEniNoMPwAAYuz9_1mcdex8MAXndPyIOAXndPyFyAO1OAA', + 'thumbnail': 'https://s.sc-cdn.net/p9mgs8YyMZzfFRls4HGZxwXWBGVF-cEa6ZYPuCT2xPk=/default/preview.jpg', + 'description': '#spotlight', + 'timestamp': 1622914559, + 'upload_date': '20210605', + 'view_count': 62100, + 'uploader_id': None + } + } + + def _real_extract(self, url): + video_id = self._match_id(url) + webpage = self._download_webpage(url, video_id) + + schema_video_object_raw = self._html_search_regex(r'', + webpage, 'schema_video_object') + schema_video_object = self._parse_json(schema_video_object_raw, video_id, fatal=True) + video_url = schema_video_object.get('contentUrl') + title = schema_video_object.get('name') + if not title: + matching = re.match(r'sc-cdn.net/\w+/(\w+)', video_url) + if matching: + title = matching.group(1) + if not title: + title = video_id + + try: + views = schema_video_object.get('interactionStatistic').get('userInteractionCount') + except AttributeError: + views = None + + try: + uploader_id = schema_video_object.get('creator').get('alternateName') + except AttributeError: + uploader_id = None + + return { + 'id': video_id, + 'title': title, + 'url': video_url, + 'ext': 'mp4', + 'thumbnail': schema_video_object.get('thumbnailUrl'), + 'timestamp': unified_timestamp(schema_video_object.get('uploadDate')), + 'description': schema_video_object.get('description'), + 'duration': parse_duration(schema_video_object.get('duration')), + 'view_count': views, + 'uploader_id': uploader_id + } From 80bc41f018ccc14c1753abe50f350793b5f3cf01 Mon Sep 17 00:00:00 2001 From: Fynn Date: Fri, 10 Sep 2021 18:45:31 +0200 Subject: [PATCH 2/2] [snapchat] Improve robustness --- youtube_dl/extractor/snapchat.py | 33 ++++++++++++++------------------ 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/youtube_dl/extractor/snapchat.py b/youtube_dl/extractor/snapchat.py index 37389d5d6..d784fbeb5 100644 --- a/youtube_dl/extractor/snapchat.py +++ b/youtube_dl/extractor/snapchat.py @@ -1,8 +1,7 @@ # coding: utf-8 from __future__ import unicode_literals -import re -from ..utils import unified_timestamp, parse_duration +from ..utils import unified_timestamp, parse_duration, try_get, str_or_none, ExtractorError from .common import InfoExtractor @@ -15,11 +14,11 @@ class SnapchatIE(InfoExtractor): 'id': 'W7_EDlXWTBiXAEEniNoMPwAAYuz9_1mcdex8MAXndPyIOAXndPyFyAO1OAA', 'ext': 'mp4', 'title': 'W7_EDlXWTBiXAEEniNoMPwAAYuz9_1mcdex8MAXndPyIOAXndPyFyAO1OAA', - 'thumbnail': 'https://s.sc-cdn.net/p9mgs8YyMZzfFRls4HGZxwXWBGVF-cEa6ZYPuCT2xPk=/default/preview.jpg', + 'thumbnail': r're:https://s\.sc-cdn\.net/.+\.jpg', 'description': '#spotlight', 'timestamp': 1622914559, 'upload_date': '20210605', - 'view_count': 62100, + 'view_count': 72100, 'uploader_id': None } } @@ -28,27 +27,23 @@ class SnapchatIE(InfoExtractor): video_id = self._match_id(url) webpage = self._download_webpage(url, video_id) - schema_video_object_raw = self._html_search_regex(r'', + schema_video_object_raw = self._html_search_regex(r']*?data-react-helmet\s*=\s*"true"\s[^>]*?type\s*=\s*"application/ld\+json">(.+?)', webpage, 'schema_video_object') schema_video_object = self._parse_json(schema_video_object_raw, video_id, fatal=True) - video_url = schema_video_object.get('contentUrl') + try: + video_url = str_or_none(schema_video_object['contentUrl']) + if not video_url: + raise ValueError('video_url must be non-empty string') + except (TypeError, ValueError) as e: + raise ExtractorError('Unexpected format for schema_video_object', cause=e, video_id=video_id) + title = schema_video_object.get('name') if not title: - matching = re.match(r'sc-cdn.net/\w+/(\w+)', video_url) - if matching: - title = matching.group(1) - if not title: - title = video_id + title = self._generic_title(url) - try: - views = schema_video_object.get('interactionStatistic').get('userInteractionCount') - except AttributeError: - views = None + views = try_get(schema_video_object.get('interactionStatistic'), lambda x: x['userInteractionCount']) - try: - uploader_id = schema_video_object.get('creator').get('alternateName') - except AttributeError: - uploader_id = None + uploader_id = try_get(schema_video_object.get('creator'), lambda x: x['alternateName']) return { 'id': video_id,