[yt-dlp.git] / yt_dlp / extractor / microsoftvirtualacademy.py

import re

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    parse_duration,
    smuggle_url,
    unsmuggle_url,
    xpath_text,
)


class MicrosoftVirtualAcademyBaseIE(InfoExtractor):
    def _extract_base_url(self, course_id, display_id):
        return self._download_json(
            'https://api-mlxprod.microsoft.com/services/products/anonymous/%s' % course_id,
            display_id, 'Downloading course base URL')

    def _extract_chapter_and_title(self, title):
        if not title:
            return None, None
        m = re.search(r'(?P<chapter>\d+)\s*\|\s*(?P<title>.+)', title)
        return (int(m.group('chapter')), m.group('title')) if m else (None, title)


class MicrosoftVirtualAcademyIE(MicrosoftVirtualAcademyBaseIE):
    IE_NAME = 'mva'
    IE_DESC = 'Microsoft Virtual Academy videos'
    _VALID_URL = r'(?:%s:|https?://(?:mva\.microsoft|(?:www\.)?microsoftvirtualacademy)\.com/[^/]+/training-courses/[^/?#&]+-)(?P<course_id>\d+)(?::|\?l=)(?P<id>[\da-zA-Z]+_\d+)' % IE_NAME

    _TESTS = [{
        'url': 'https://mva.microsoft.com/en-US/training-courses/microsoft-azure-fundamentals-virtual-machines-11788?l=gfVXISmEB_6804984382',
        'md5': '7826c44fc31678b12ad8db11f6b5abb9',
        'info_dict': {
            'id': 'gfVXISmEB_6804984382',
            'ext': 'mp4',
            'title': 'Course Introduction',
            'formats': 'mincount:3',
            'subtitles': {
                'en': [{
                    'ext': 'ttml',
                }],
            },
        }
    }, {
        'url': 'mva:11788:gfVXISmEB_6804984382',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        url, smuggled_data = unsmuggle_url(url, {})

        mobj = self._match_valid_url(url)
        course_id = mobj.group('course_id')
        video_id = mobj.group('id')

        base_url = smuggled_data.get('base_url') or self._extract_base_url(course_id, video_id)

        settings = self._download_xml(
            '%s/content/content_%s/videosettings.xml?v=1' % (base_url, video_id),
            video_id, 'Downloading video settings XML')

        _, title = self._extract_chapter_and_title(xpath_text(
            settings, './/Title', 'title', fatal=True))

        formats = []

        for sources in settings.findall('.//MediaSources'):
            sources_type = sources.get('videoType')
            for source in sources.findall('./MediaSource'):
                video_url = source.text
                if not video_url or not video_url.startswith('http'):
                    continue
                if sources_type == 'smoothstreaming':
                    formats.extend(self._extract_ism_formats(
                        video_url, video_id, 'mss', fatal=False))
                    continue
                video_mode = source.get('videoMode')
                height = int_or_none(self._search_regex(
                    r'^(\d+)[pP]$', video_mode or '', 'height', default=None))
                codec = source.get('codec')
                acodec, vcodec = [None] * 2
                if codec:
                    codecs = codec.split(',')
                    if len(codecs) == 2:
                        acodec, vcodec = codecs
                    elif len(codecs) == 1:
                        vcodec = codecs[0]
                formats.append({
                    'url': video_url,
                    'format_id': video_mode,
                    'height': height,
                    'acodec': acodec,
                    'vcodec': vcodec,
                })

        subtitles = {}
        for source in settings.findall('.//MarkerResourceSource'):
            subtitle_url = source.text
            if not subtitle_url:
                continue
            subtitles.setdefault('en', []).append({
                'url': '%s/%s' % (base_url, subtitle_url),
                'ext': source.get('type'),
            })

        return {
            'id': video_id,
            'title': title,
            'subtitles': subtitles,
            'formats': formats
        }


class MicrosoftVirtualAcademyCourseIE(MicrosoftVirtualAcademyBaseIE):
    IE_NAME = 'mva:course'
    IE_DESC = 'Microsoft Virtual Academy courses'
    _VALID_URL = r'(?:%s:|https?://(?:mva\.microsoft|(?:www\.)?microsoftvirtualacademy)\.com/[^/]+/training-courses/(?P<display_id>[^/?#&]+)-)(?P<id>\d+)' % IE_NAME

    _TESTS = [{
        'url': 'https://mva.microsoft.com/en-US/training-courses/microsoft-azure-fundamentals-virtual-machines-11788',
        'info_dict': {
            'id': '11788',
            'title': 'Microsoft Azure Fundamentals: Virtual Machines',
        },
        'playlist_count': 36,
    }, {
        # with emphasized chapters
        'url': 'https://mva.microsoft.com/en-US/training-courses/developing-windows-10-games-with-construct-2-16335',
        'info_dict': {
            'id': '16335',
            'title': 'Developing Windows 10 Games with Construct 2',
        },
        'playlist_count': 10,
    }, {
        'url': 'https://www.microsoftvirtualacademy.com/en-US/training-courses/microsoft-azure-fundamentals-virtual-machines-11788',
        'only_matching': True,
    }, {
        'url': 'mva:course:11788',
        'only_matching': True,
    }]

    @classmethod
    def suitable(cls, url):
        return False if MicrosoftVirtualAcademyIE.suitable(url) else super(
            MicrosoftVirtualAcademyCourseIE, cls).suitable(url)

    def _real_extract(self, url):
        mobj = self._match_valid_url(url)
        course_id = mobj.group('id')
        display_id = mobj.group('display_id')

        base_url = self._extract_base_url(course_id, display_id)

        manifest = self._download_json(
            '%s/imsmanifestlite.json' % base_url,
            display_id, 'Downloading course manifest JSON')['manifest']

        organization = manifest['organizations']['organization'][0]

        entries = []
        for chapter in organization['item']:
            chapter_number, chapter_title = self._extract_chapter_and_title(chapter.get('title'))
            chapter_id = chapter.get('@identifier')
            for item in chapter.get('item', []):
                item_id = item.get('@identifier')
                if not item_id:
                    continue
                metadata = item.get('resource', {}).get('metadata') or {}
                if metadata.get('learningresourcetype') != 'Video':
                    continue
                _, title = self._extract_chapter_and_title(item.get('title'))
                duration = parse_duration(metadata.get('duration'))
                description = metadata.get('description')
                entries.append({
                    '_type': 'url_transparent',
                    'url': smuggle_url(
                        'mva:%s:%s' % (course_id, item_id), {'base_url': base_url}),
                    'title': title,
                    'description': description,
                    'duration': duration,
                    'chapter': chapter_title,
                    'chapter_number': chapter_number,
                    'chapter_id': chapter_id,
                })

        title = organization.get('title') or manifest.get('metadata', {}).get('title')

        return self.playlist_result(entries, course_id, title)
Commit	Line	Data
f23a92a0 S	1	import re
	2
	3	from .common import InfoExtractor
f23a92a0 S	4	from ..utils import (
	5	int_or_none,
	6	parse_duration,
	7	smuggle_url,
	8	unsmuggle_url,
	9	xpath_text,
	10	)
	11
	12
	13	class MicrosoftVirtualAcademyBaseIE(InfoExtractor):
	14	def _extract_base_url(self, course_id, display_id):
	15	return self._download_json(
	16	'https://api-mlxprod.microsoft.com/services/products/anonymous/%s' % course_id,
	17	display_id, 'Downloading course base URL')
	18
	19	def _extract_chapter_and_title(self, title):
	20	if not title:
	21	return None, None
	22	m = re.search(r'(?P<chapter>\d+)\s\\|\s(?P<title>.+)', title)
	23	return (int(m.group('chapter')), m.group('title')) if m else (None, title)
	24
	25
	26	class MicrosoftVirtualAcademyIE(MicrosoftVirtualAcademyBaseIE):
	27	IE_NAME = 'mva'
	28	IE_DESC = 'Microsoft Virtual Academy videos'
c52f4efa	29	_VALID_URL = r'(?:%s:\|https?://(?:mva\.microsoft\|(?:www\.)?microsoftvirtualacademy)\.com/[^/]+/training-courses/[^/?#&]+-)(?P<course_id>\d+)(?::\|\?l=)(?P<id>[\da-zA-Z]+_\d+)' % IE_NAME
f23a92a0 S	30
	31	_TESTS = [{
	32	'url': 'https://mva.microsoft.com/en-US/training-courses/microsoft-azure-fundamentals-virtual-machines-11788?l=gfVXISmEB_6804984382',
	33	'md5': '7826c44fc31678b12ad8db11f6b5abb9',
	34	'info_dict': {
	35	'id': 'gfVXISmEB_6804984382',
	36	'ext': 'mp4',
	37	'title': 'Course Introduction',
	38	'formats': 'mincount:3',
	39	'subtitles': {
	40	'en': [{
	41	'ext': 'ttml',
	42	}],
	43	},
	44	}
	45	}, {
	46	'url': 'mva:11788:gfVXISmEB_6804984382',
	47	'only_matching': True,
	48	}]
	49
	50	def _real_extract(self, url):
	51	url, smuggled_data = unsmuggle_url(url, {})
	52
5ad28e7f	53	mobj = self._match_valid_url(url)
f23a92a0 S	54	course_id = mobj.group('course_id')
	55	video_id = mobj.group('id')
	56
	57	base_url = smuggled_data.get('base_url') or self._extract_base_url(course_id, video_id)
	58
	59	settings = self._download_xml(
	60	'%s/content/content_%s/videosettings.xml?v=1' % (base_url, video_id),
	61	video_id, 'Downloading video settings XML')
	62
	63	_, title = self._extract_chapter_and_title(xpath_text(
	64	settings, './/Title', 'title', fatal=True))
	65
	66	formats = []
	67
f9934b96	68	for sources in settings.findall('.//MediaSources'):
639e3b5c	69	sources_type = sources.get('videoType')
f9934b96	70	for source in sources.findall('./MediaSource'):
f23a92a0 S	71	video_url = source.text
	72	if not video_url or not video_url.startswith('http'):
	73	continue
639e3b5c RA	74	if sources_type == 'smoothstreaming':
	75	formats.extend(self._extract_ism_formats(
	76	video_url, video_id, 'mss', fatal=False))
	77	continue
f23a92a0 S	78	video_mode = source.get('videoMode')
	79	height = int_or_none(self._search_regex(
	80	r'^(\d+)[pP]$', video_mode or '', 'height', default=None))
	81	codec = source.get('codec')
	82	acodec, vcodec = [None] * 2
	83	if codec:
	84	codecs = codec.split(',')
	85	if len(codecs) == 2:
	86	acodec, vcodec = codecs
	87	elif len(codecs) == 1:
	88	vcodec = codecs[0]
	89	formats.append({
	90	'url': video_url,
	91	'format_id': video_mode,
	92	'height': height,
	93	'acodec': acodec,
	94	'vcodec': vcodec,
	95	})
f23a92a0 S	96
f23a92a0 S	97	subtitles = {}
f9934b96	98	for source in settings.findall('.//MarkerResourceSource'):
f23a92a0 S	99	subtitle_url = source.text
	100	if not subtitle_url:
	101	continue
	102	subtitles.setdefault('en', []).append({
	103	'url': '%s/%s' % (base_url, subtitle_url),
	104	'ext': source.get('type'),
	105	})
	106
	107	return {
	108	'id': video_id,
	109	'title': title,
	110	'subtitles': subtitles,
	111	'formats': formats
	112	}
	113
	114
	115	class MicrosoftVirtualAcademyCourseIE(MicrosoftVirtualAcademyBaseIE):
	116	IE_NAME = 'mva:course'
	117	IE_DESC = 'Microsoft Virtual Academy courses'
c52f4efa	118	_VALID_URL = r'(?:%s:\|https?://(?:mva\.microsoft\|(?:www\.)?microsoftvirtualacademy)\.com/[^/]+/training-courses/(?P<display_id>[^/?#&]+)-)(?P<id>\d+)' % IE_NAME
f23a92a0 S	119
	120	_TESTS = [{
	121	'url': 'https://mva.microsoft.com/en-US/training-courses/microsoft-azure-fundamentals-virtual-machines-11788',
	122	'info_dict': {
	123	'id': '11788',
	124	'title': 'Microsoft Azure Fundamentals: Virtual Machines',
	125	},
	126	'playlist_count': 36,
	127	}, {
	128	# with emphasized chapters
	129	'url': 'https://mva.microsoft.com/en-US/training-courses/developing-windows-10-games-with-construct-2-16335',
	130	'info_dict': {
	131	'id': '16335',
	132	'title': 'Developing Windows 10 Games with Construct 2',
	133	},
	134	'playlist_count': 10,
	135	}, {
	136	'url': 'https://www.microsoftvirtualacademy.com/en-US/training-courses/microsoft-azure-fundamentals-virtual-machines-11788',
	137	'only_matching': True,
	138	}, {
	139	'url': 'mva:course:11788',
	140	'only_matching': True,
	141	}]
	142
	143	@classmethod
	144	def suitable(cls, url):
	145	return False if MicrosoftVirtualAcademyIE.suitable(url) else super(
	146	MicrosoftVirtualAcademyCourseIE, cls).suitable(url)
	147
	148	def _real_extract(self, url):
5ad28e7f	149	mobj = self._match_valid_url(url)
f23a92a0 S	150	course_id = mobj.group('id')
	151	display_id = mobj.group('display_id')
	152
	153	base_url = self._extract_base_url(course_id, display_id)
	154
	155	manifest = self._download_json(
	156	'%s/imsmanifestlite.json' % base_url,
	157	display_id, 'Downloading course manifest JSON')['manifest']
	158
	159	organization = manifest['organizations']['organization'][0]
	160
	161	entries = []
	162	for chapter in organization['item']:
	163	chapter_number, chapter_title = self._extract_chapter_and_title(chapter.get('title'))
	164	chapter_id = chapter.get('@identifier')
	165	for item in chapter.get('item', []):
	166	item_id = item.get('@identifier')
	167	if not item_id:
	168	continue
	169	metadata = item.get('resource', {}).get('metadata') or {}
	170	if metadata.get('learningresourcetype') != 'Video':
	171	continue
	172	_, title = self._extract_chapter_and_title(item.get('title'))
	173	duration = parse_duration(metadata.get('duration'))
	174	description = metadata.get('description')
	175	entries.append({
	176	'_type': 'url_transparent',
	177	'url': smuggle_url(
	178	'mva:%s:%s' % (course_id, item_id), {'base_url': base_url}),
	179	'title': title,
	180	'description': description,
	181	'duration': duration,
	182	'chapter': chapter_title,
	183	'chapter_number': chapter_number,
	184	'chapter_id': chapter_id,
	185	})
	186
	187	title = organization.get('title') or manifest.get('metadata', {}).get('title')
	188
	189	return self.playlist_result(entries, course_id, title)