import re import urllib.parse from .common import InfoExtractor from ..utils import ( ExtractorError, parse_duration, url_or_none, urlencode_postdata, ) class FrontendMastersBaseIE(InfoExtractor): _API_BASE = 'https://api.frontendmasters.com/v1/kabuki' _LOGIN_URL = 'https://frontendmasters.com/login/' _NETRC_MACHINE = 'frontendmasters' _QUALITIES = { 'low': {'width': 480, 'height': 360}, 'mid': {'width': 1280, 'height': 720}, 'high': {'width': 1920, 'height': 1080}, } def _perform_login(self, username, password): login_page = self._download_webpage( self._LOGIN_URL, None, 'Downloading login page') login_form = self._hidden_inputs(login_page) login_form.update({ 'username': username, 'password': password, }) post_url = self._search_regex( r']+action=(["\'])(?P.+?)\1', login_page, 'post_url', default=self._LOGIN_URL, group='url') if not post_url.startswith('http'): post_url = urllib.parse.urljoin(self._LOGIN_URL, post_url) response = self._download_webpage( post_url, None, 'Logging in', data=urlencode_postdata(login_form), headers={'Content-Type': 'application/x-www-form-urlencoded'}) # Successful login if any(p in response for p in ( 'wp-login.php?action=logout', '>Logout')): return error = self._html_search_regex( r'class=(["\'])(?:(?!\1).)*\bMessageAlert\b(?:(?!\1).)*\1[^>]*>(?P[^<]+)<', response, 'error message', default=None, group='error') if error: raise ExtractorError(f'Unable to login: {error}', expected=True) raise ExtractorError('Unable to log in') class FrontendMastersPageBaseIE(FrontendMastersBaseIE): def _download_course(self, course_name, url): return self._download_json( f'{self._API_BASE}/courses/{course_name}', course_name, 'Downloading course JSON', headers={'Referer': url}) @staticmethod def _extract_chapters(course): chapters = [] lesson_elements = course.get('lessonElements') if isinstance(lesson_elements, list): chapters = [url_or_none(e) for e in lesson_elements if url_or_none(e)] return chapters @staticmethod def _extract_lesson(chapters, lesson_id, lesson): title = lesson.get('title') or lesson_id display_id = lesson.get('slug') description = lesson.get('description') thumbnail = lesson.get('thumbnail') chapter_number = None index = lesson.get('index') element_index = lesson.get('elementIndex') if (isinstance(index, int) and isinstance(element_index, int) and index < element_index): chapter_number = element_index - index chapter = (chapters[chapter_number - 1] if chapter_number - 1 < len(chapters) else None) duration = None timestamp = lesson.get('timestamp') if isinstance(timestamp, str): mobj = re.search( r'(?P\d{1,2}:\d{1,2}:\d{1,2})\s*-(?P\s*\d{1,2}:\d{1,2}:\d{1,2})', timestamp) if mobj: duration = parse_duration(mobj.group('end')) - parse_duration( mobj.group('start')) return { '_type': 'url_transparent', 'url': f'frontendmasters:{lesson_id}', 'ie_key': FrontendMastersIE.ie_key(), 'id': lesson_id, 'display_id': display_id, 'title': title, 'description': description, 'thumbnail': thumbnail, 'duration': duration, 'chapter': chapter, 'chapter_number': chapter_number, } class FrontendMastersIE(FrontendMastersBaseIE): _VALID_URL = r'(?:frontendmasters:|https?://api\.frontendmasters\.com/v\d+/kabuki/video/)(?P[^/]+)' _TESTS = [{ 'url': 'https://api.frontendmasters.com/v1/kabuki/video/a2qogef6ba', 'md5': '7f161159710d6b7016a4f4af6fcb05e2', 'info_dict': { 'id': 'a2qogef6ba', 'ext': 'mp4', 'title': 'a2qogef6ba', }, 'skip': 'Requires FrontendMasters account credentials', }, { 'url': 'frontendmasters:a2qogef6ba', 'only_matching': True, }] def _real_extract(self, url): lesson_id = self._match_id(url) source_url = f'{self._API_BASE}/video/{lesson_id}/source' formats = [] for ext in ('webm', 'mp4'): for quality in ('low', 'mid', 'high'): resolution = self._QUALITIES[quality].copy() format_id = f'{ext}-{quality}' format_url = self._download_json( source_url, lesson_id, f'Downloading {format_id} source JSON', query={ 'f': ext, 'r': resolution['height'], }, headers={ 'Referer': url, }, fatal=False)['url'] if not format_url: continue f = resolution.copy() f.update({ 'url': format_url, 'ext': ext, 'format_id': format_id, }) formats.append(f) subtitles = { 'en': [{ 'url': f'{self._API_BASE}/transcripts/{lesson_id}.vtt', }], } return { 'id': lesson_id, 'title': lesson_id, 'formats': formats, 'subtitles': subtitles, } class FrontendMastersLessonIE(FrontendMastersPageBaseIE): _VALID_URL = r'https?://(?:www\.)?frontendmasters\.com/courses/(?P[^/]+)/(?P[^/]+)' _TEST = { 'url': 'https://frontendmasters.com/courses/web-development/tools', 'info_dict': { 'id': 'a2qogef6ba', 'display_id': 'tools', 'ext': 'mp4', 'title': 'Tools', 'description': 'md5:82c1ea6472e88ed5acd1829fe992e4f7', 'thumbnail': r're:^https?://.*\.jpg$', 'chapter': 'Introduction', 'chapter_number': 1, }, 'params': { 'skip_download': True, }, 'skip': 'Requires FrontendMasters account credentials', } def _real_extract(self, url): mobj = self._match_valid_url(url) course_name, lesson_name = mobj.group('course_name', 'lesson_name') course = self._download_course(course_name, url) lesson_id, lesson = next( (video_id, data) for video_id, data in course['lessonData'].items() if data.get('slug') == lesson_name) chapters = self._extract_chapters(course) return self._extract_lesson(chapters, lesson_id, lesson) class FrontendMastersCourseIE(FrontendMastersPageBaseIE): _VALID_URL = r'https?://(?:www\.)?frontendmasters\.com/courses/(?P[^/]+)' _TEST = { 'url': 'https://frontendmasters.com/courses/web-development/', 'info_dict': { 'id': 'web-development', 'title': 'Introduction to Web Development', 'description': 'md5:9317e6e842098bf725d62360e52d49a6', }, 'playlist_count': 81, 'skip': 'Requires FrontendMasters account credentials', } @classmethod def suitable(cls, url): return False if FrontendMastersLessonIE.suitable(url) else super( FrontendMastersBaseIE, cls).suitable(url) def _real_extract(self, url): course_name = self._match_id(url) course = self._download_course(course_name, url) chapters = self._extract_chapters(course) lessons = sorted( course['lessonData'].values(), key=lambda data: data['index']) entries = [] for lesson in lessons: lesson_name = lesson.get('slug') lesson_id = lesson.get('hash') or lesson.get('statsId') if not lesson_id or not lesson_name: continue entries.append(self._extract_lesson(chapters, lesson_id, lesson)) title = course.get('title') description = course.get('description') return self.playlist_result(entries, course_name, title, description)