release 2016.10.07

[ci skip]

.github/ISSUE_TEMPLATE.md

@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.10.02*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.10.02**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.10.07*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.10.07**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.10.02
+[debug] youtube-dl version 2016.10.07
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}

AUTHORS

@@ -26,7 +26,7 @@ Albert Kim
 Pierre Rudloff
 Huarong Huo
 Ismael Mejía
-Steffan 'Ruirize' James
+Steffan Donal
 Andras Elso
 Jelle van der Waa
 Marcin Cieślak

ChangeLog

@@ -1,3 +1,23 @@
+version 2016.10.07
+
+Extractors
++ [iprima] Detect geo restriction
+* [facebook] Fix video extraction (#10846)
++ [commonprotocols] Support direct MMS links (#10838)
++ [generic] Add support for multiple vimeo embeds (#10862)
++ [nzz] Add support for nzz.ch (#4407)
++ [npo] Detect geo restriction
++ [npo] Add support for 2doc.nl (#10842)
++ [lego] Add support for lego.com (#10369)
++ [tonline] Add support for t-online.de (#10376)
+* [techtalks] Relax URL regular expression (#10840)
+* [youtube:live] Extend URL regular expression (#10839)
++ [theweatherchannel] Add support for weather.com (#7188)
++ [thisoldhouse] Add support for thisoldhouse.com (#10837)
++ [nhl] Add support for wch2016.com (#10833)
+* [pornoxo] Use JWPlatform to improve metadata extraction
+
+
 version 2016.10.02
 
 Core

devscripts/lazy_load_template.py

@@ -1,4 +1,4 @@
-# encoding: utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re

docs/conf.py

@@ -1,4 +1,4 @@
-# -*- coding: utf-8 -*-
+# coding: utf-8
 #
 # youtube-dl documentation build configuration file, created by
 # sphinx-quickstart on Fri Mar 14 21:05:43 2014.

docs/supportedsites.md

@@ -364,6 +364,7 @@
  - **Le**: 乐视网
  - **Learnr**
  - **Lecture2Go**
+ - **LEGO**
  - **Lemonde**
  - **LePlaylist**
  - **LetvCloud**: 乐视云
@@ -507,6 +508,7 @@
  - **Nuvid**
  - **NYTimes**
  - **NYTimesArticle**
+ - **NZZ**
  - **ocw.mit.edu**
  - **OdaTV**
  - **Odnoklassniki**
@@ -692,6 +694,7 @@
  - **SWRMediathek**
  - **Syfy**
  - **SztvHu**
+ - **t-online.de**
  - **Tagesschau**
  - **tagesschau:player**
  - **Tass**
@@ -721,8 +724,10 @@
  - **TheScene**
  - **TheSixtyOne**
  - **TheStar**
+ - **TheWeatherChannel**
  - **ThisAmericanLife**
  - **ThisAV**
+ - **ThisOldHouse**
  - **tinypic**: tinypic.com videos
  - **tlc.de**
  - **TMZ**

setup.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-# -*- coding: utf-8 -*-
+# coding: utf-8
 
 from __future__ import print_function
 

youtube_dl/YoutubeDL.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-# -*- coding: utf-8 -*-
+# coding: utf-8
 
 from __future__ import absolute_import, unicode_literals
 

youtube_dl/__init__.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-# -*- coding: utf-8 -*-
+# coding: utf-8
 
 from __future__ import unicode_literals
 

youtube_dl/extractor/adobepass.py

@@ -1,4 +1,4 @@
-# -*- coding: utf-8 -*-
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re

youtube_dl/extractor/allocine.py

@@ -1,4 +1,4 @@
-# -*- coding: utf-8 -*-
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re

youtube_dl/extractor/arte.py

@@ -1,4 +1,4 @@
-# encoding: utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re

youtube_dl/extractor/brightcove.py

@@ -1,4 +1,4 @@
-# encoding: utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re

youtube_dl/extractor/canalplus.py

@@ -1,4 +1,4 @@
-# encoding: utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re

youtube_dl/extractor/cbsnews.py

@@ -1,4 +1,4 @@
-# encoding: utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
 from .common import InfoExtractor

youtube_dl/extractor/ceskatelevize.py

@@ -1,4 +1,4 @@
-# -*- coding: utf-8 -*-
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re

youtube_dl/extractor/comcarcoff.py

@@ -1,4 +1,4 @@
-# encoding: utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
 from .common import InfoExtractor

youtube_dl/extractor/common.py

@@ -21,6 +21,7 @@ from ..compat import (
     compat_os_name,
     compat_str,
     compat_urllib_error,
+    compat_urllib_parse_unquote,
     compat_urllib_parse_urlencode,
     compat_urllib_request,
     compat_urlparse,
@@ -2020,6 +2021,12 @@ class InfoExtractor(object):
             headers['Ytdl-request-proxy'] = geo_verification_proxy
         return headers
 
+    def _generic_id(self, url):
+        return compat_urllib_parse_unquote(os.path.splitext(url.rstrip('/').split('/')[-1])[0])
+
+    def _generic_title(self, url):
+        return compat_urllib_parse_unquote(os.path.splitext(url_basename(url))[0])
+
 
 class SearchInfoExtractor(InfoExtractor):
     """

youtube_dl/extractor/commonprotocols.py

@@ -1,13 +1,9 @@
 from __future__ import unicode_literals
 
-import os
-
 from .common import InfoExtractor
 from ..compat import (
-    compat_urllib_parse_unquote,
     compat_urlparse,
 )
-from ..utils import url_basename
 
 
 class RtmpIE(InfoExtractor):
@@ -23,8 +19,8 @@ class RtmpIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        video_id = compat_urllib_parse_unquote(os.path.splitext(url.rstrip('/').split('/')[-1])[0])
-        title = compat_urllib_parse_unquote(os.path.splitext(url_basename(url))[0])
+        video_id = self._generic_id(url)
+        title = self._generic_title(url)
         return {
             'id': video_id,
             'title': title,
@@ -34,3 +30,31 @@ class RtmpIE(InfoExtractor):
                 'format_id': compat_urlparse.urlparse(url).scheme,
             }],
         }
+
+
+class MmsIE(InfoExtractor):
+    IE_DESC = False  # Do not list
+    _VALID_URL = r'(?i)mms://.+'
+
+    _TEST = {
+        # Direct MMS link
+        'url': 'mms://kentro.kaist.ac.kr/200907/MilesReid(0709).wmv',
+        'info_dict': {
+            'id': 'MilesReid(0709)',
+            'ext': 'wmv',
+            'title': 'MilesReid(0709)',
+        },
+        'params': {
+            'skip_download': True,  # rtsp downloads, requiring mplayer or mpv
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._generic_id(url)
+        title = self._generic_title(url)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'url': url,
+        }

youtube_dl/extractor/crunchyroll.py

@@ -1,4 +1,4 @@
-# encoding: utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re

youtube_dl/extractor/daum.py

@@ -1,4 +1,4 @@
-# encoding: utf-8
+# coding: utf-8
 
 from __future__ import unicode_literals
 

youtube_dl/extractor/dramafever.py

@@ -1,4 +1,4 @@
-# encoding: utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
 import itertools

youtube_dl/extractor/eitb.py

@@ -1,4 +1,4 @@
-# encoding: utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
 from .common import InfoExtractor

youtube_dl/extractor/embedly.py

@@ -1,4 +1,4 @@
-# encoding: utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
 from .common import InfoExtractor

youtube_dl/extractor/extractors.py

@@ -186,7 +186,10 @@ from .comedycentral import (
 )
 from .comcarcoff import ComCarCoffIE
 from .commonmistakes import CommonMistakesIE, UnicodeBOMIE
-from .commonprotocols import RtmpIE
+from .commonprotocols import (
+    MmsIE,
+    RtmpIE,
+)
 from .condenast import CondeNastIE
 from .cracked import CrackedIE
 from .crackle import CrackleIE
@@ -437,6 +440,7 @@ from .lcp import (
 )
 from .learnr import LearnrIE
 from .lecture2go import Lecture2GoIE
+from .lego import LEGOIE
 from .lemonde import LemondeIE
 from .leeco import (
     LeIE,
@@ -637,6 +641,7 @@ from .nytimes import (
     NYTimesArticleIE,
 )
 from .nuvid import NuvidIE
+from .nzz import NZZIE
 from .odatv import OdaTVIE
 from .odnoklassniki import OdnoklassnikiIE
 from .oktoberfesttv import OktoberfestTVIE
@@ -890,8 +895,10 @@ from .theplatform import (
 from .thescene import TheSceneIE
 from .thesixtyone import TheSixtyOneIE
 from .thestar import TheStarIE
+from .theweatherchannel import TheWeatherChannelIE
 from .thisamericanlife import ThisAmericanLifeIE
 from .thisav import ThisAVIE
+from .thisoldhouse import ThisOldHouseIE
 from .threeqsdn import ThreeQSDNIE
 from .tinypic import TinyPicIE
 from .tlc import TlcDeIE
@@ -906,6 +913,7 @@ from .tnaflix import (
     MovieFapIE,
 )
 from .toggle import ToggleIE
+from .tonline import TOnlineIE
 from .toutv import TouTvIE
 from .toypics import ToypicsUserIE, ToypicsIE
 from .traileraddict import TrailerAddictIE

youtube_dl/extractor/facebook.py

@@ -258,7 +258,7 @@ class FacebookIE(InfoExtractor):
 
         if not video_data:
             server_js_data = self._parse_json(self._search_regex(
-                r'handleServerJS\(({.+})\);', webpage, 'server js data', default='{}'), video_id)
+                r'handleServerJS\(({.+})(?:\);|,")', webpage, 'server js data', default='{}'), video_id)
             for item in server_js_data.get('instances', []):
                 if item[1][0] == 'VideoConfig':
                     video_data = video_data_list2dict(item[2][0]['videoData'])

youtube_dl/extractor/faz.py

@@ -1,4 +1,4 @@
-# encoding: utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
 from .common import InfoExtractor

youtube_dl/extractor/firsttv.py

@@ -1,4 +1,4 @@
-# encoding: utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
 from .common import InfoExtractor

youtube_dl/extractor/folketinget.py

@@ -1,4 +1,4 @@
-# -*- coding: utf-8 -*-
+# coding: utf-8
 from __future__ import unicode_literals
 
 from .common import InfoExtractor

youtube_dl/extractor/francetv.py

@@ -1,4 +1,4 @@
-# encoding: utf-8
+# coding: utf-8
 
 from __future__ import unicode_literals
 

youtube_dl/extractor/generic.py

@@ -1,4 +1,4 @@
-# encoding: utf-8
+# coding: utf-8
 
 from __future__ import unicode_literals
 
@@ -27,7 +27,6 @@ from ..utils import (
     unified_strdate,
     unsmuggle_url,
     UnsupportedError,
-    url_basename,
     xpath_text,
 )
 from .brightcove import (
@@ -1549,7 +1548,7 @@ class GenericIE(InfoExtractor):
             force_videoid = smuggled_data['force_videoid']
             video_id = force_videoid
         else:
-            video_id = compat_urllib_parse_unquote(os.path.splitext(url.rstrip('/').split('/')[-1])[0])
+            video_id = self._generic_id(url)
 
         self.to_screen('%s: Requesting header' % video_id)
 
@@ -1578,7 +1577,7 @@ class GenericIE(InfoExtractor):
 
         info_dict = {
             'id': video_id,
-            'title': compat_urllib_parse_unquote(os.path.splitext(url_basename(url))[0]),
+            'title': self._generic_title(url),
             'upload_date': unified_strdate(head_response.headers.get('Last-Modified'))
         }
 
@@ -1754,9 +1753,9 @@ class GenericIE(InfoExtractor):
         if matches:
             return _playlist_from_matches(matches, ie='RtlNl')
 
-        vimeo_url = VimeoIE._extract_vimeo_url(url, webpage)
-        if vimeo_url is not None:
-            return self.url_result(vimeo_url)
+        vimeo_urls = VimeoIE._extract_urls(url, webpage)
+        if vimeo_urls:
+            return _playlist_from_matches(vimeo_urls, ie=VimeoIE.ie_key())
 
         vid_me_embed_url = self._search_regex(
             r'src=[\'"](https?://vid\.me/[^\'"]+)[\'"]',

youtube_dl/extractor/goshgay.py

@@ -1,4 +1,4 @@
-# -*- coding: utf-8 -*-
+# coding: utf-8
 from __future__ import unicode_literals
 
 from .common import InfoExtractor

youtube_dl/extractor/hark.py

@@ -1,4 +1,4 @@
-# -*- coding: utf-8 -*-
+# coding: utf-8
 from __future__ import unicode_literals
 
 from .common import InfoExtractor

youtube_dl/extractor/helsinki.py

@@ -1,4 +1,4 @@
-# -*- coding: utf-8 -*-
+# coding: utf-8
 
 from __future__ import unicode_literals
 

youtube_dl/extractor/ina.py

@@ -1,4 +1,4 @@
-# encoding: utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re

youtube_dl/extractor/iprima.py

@@ -81,6 +81,9 @@ class IPrimaIE(InfoExtractor):
             for _, src in re.findall(r'src["\']\s*:\s*(["\'])(.+?)\1', playerpage):
                 extract_formats(src)
 
+        if not formats and '>GEO_IP_NOT_ALLOWED<' in playerpage:
+            self.raise_geo_restricted()
+
         self._sort_formats(formats)
 
         return {

youtube_dl/extractor/jpopsukitv.py

@@ -1,4 +1,4 @@
-# coding=utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
 from .common import InfoExtractor

youtube_dl/extractor/kickstarter.py

@@ -1,4 +1,4 @@
-# encoding: utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
 from .common import InfoExtractor

youtube_dl/extractor/kontrtube.py

@@ -1,4 +1,4 @@
-# encoding: utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re

youtube_dl/extractor/krasview.py

@@ -1,4 +1,4 @@
-# encoding: utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
 import json

youtube_dl/extractor/lego.py

@@ -0,0 +1,86 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..compat import compat_str
+from ..utils import (
+    unescapeHTML,
+    int_or_none,
+)
+
+
+class LEGOIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?lego\.com/(?:[^/]+/)*videos/(?:[^/]+/)*[^/?#]+-(?P<id>[0-9a-f]+)'
+    _TEST = {
+        'url': 'http://www.lego.com/en-us/videos/themes/club/blocumentary-kawaguchi-55492d823b1b4d5e985787fa8c2973b1',
+        'md5': 'f34468f176cfd76488767fc162c405fa',
+        'info_dict': {
+            'id': '55492d823b1b4d5e985787fa8c2973b1',
+            'ext': 'mp4',
+            'title': 'Blocumentary Great Creations: Akiyuki Kawaguchi',
+        }
+    }
+    _BITRATES = [256, 512, 1024, 1536, 2560]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(
+            'http://www.lego.com/en-US/mediaplayer/video/' + video_id, video_id)
+        title = self._search_regex(r'<title>(.+?)</title>', webpage, 'title')
+        video_data = self._parse_json(unescapeHTML(self._search_regex(
+            r"video='([^']+)'", webpage, 'video data')), video_id)
+        progressive_base = self._search_regex(
+            r'data-video-progressive-url="([^"]+)"',
+            webpage, 'progressive base', default='https://lc-mediaplayerns-live-s.legocdn.com/')
+        streaming_base = self._search_regex(
+            r'data-video-streaming-url="([^"]+)"',
+            webpage, 'streaming base', default='http://legoprod-f.akamaihd.net/')
+        item_id = video_data['ItemId']
+
+        net_storage_path = video_data.get('NetStoragePath') or '/'.join([item_id[:2], item_id[2:4]])
+        base_path = '_'.join([item_id, video_data['VideoId'], video_data['Locale'], compat_str(video_data['VideoVersion'])])
+        path = '/'.join([net_storage_path, base_path])
+        streaming_path = ','.join(map(lambda bitrate: compat_str(bitrate), self._BITRATES))
+
+        formats = self._extract_akamai_formats(
+            '%si/s/public/%s_,%s,.mp4.csmil/master.m3u8' % (streaming_base, path, streaming_path), video_id)
+        m3u8_formats = list(filter(
+            lambda f: f.get('protocol') == 'm3u8_native' and f.get('vcodec') != 'none' and f.get('resolution') != 'multiple',
+            formats))
+        if len(m3u8_formats) == len(self._BITRATES):
+            self._sort_formats(m3u8_formats)
+            for bitrate, m3u8_format in zip(self._BITRATES, m3u8_formats):
+                progressive_base_url = '%spublic/%s_%d.' % (progressive_base, path, bitrate)
+                mp4_f = m3u8_format.copy()
+                mp4_f.update({
+                    'url': progressive_base_url + 'mp4',
+                    'format_id': m3u8_format['format_id'].replace('hls', 'mp4'),
+                    'protocol': 'http',
+                })
+                web_f = {
+                    'url': progressive_base_url + 'webm',
+                    'format_id': m3u8_format['format_id'].replace('hls', 'webm'),
+                    'width': m3u8_format['width'],
+                    'height': m3u8_format['height'],
+                    'tbr': m3u8_format.get('tbr'),
+                    'ext': 'webm',
+                }
+                formats.extend([web_f, mp4_f])
+        else:
+            for bitrate in self._BITRATES:
+                for ext in ('web', 'mp4'):
+                    formats.append({
+                        'format_id': '%s-%s' % (ext, bitrate),
+                        'url': '%spublic/%s_%d.%s' % (progressive_base, path, bitrate, ext),
+                        'tbr': bitrate,
+                        'ext': ext,
+                    })
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'thumbnail': video_data.get('CoverImageUrl'),
+            'duration': int_or_none(video_data.get('Length')),
+            'formats': formats,
+        }

youtube_dl/extractor/lifenews.py

@@ -1,4 +1,4 @@
-# encoding: utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re

youtube_dl/extractor/m6.py

@@ -1,4 +1,4 @@
-# encoding: utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
 from .common import InfoExtractor

youtube_dl/extractor/mailru.py

@@ -1,4 +1,4 @@
-# encoding: utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re

youtube_dl/extractor/moviezine.py

@@ -1,4 +1,4 @@
-# -*- coding: utf-8 -*-
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re

youtube_dl/extractor/musicplayon.py

@@ -1,4 +1,4 @@
-# encoding: utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
 from .common import InfoExtractor

youtube_dl/extractor/myspace.py

@@ -1,4 +1,4 @@
-# encoding: utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re

youtube_dl/extractor/naver.py

@@ -1,4 +1,4 @@
-# encoding: utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re

youtube_dl/extractor/newstube.py

@@ -1,4 +1,4 @@
-# encoding: utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re

youtube_dl/extractor/nhl.py

@@ -245,7 +245,11 @@ class NHLVideocenterCategoryIE(NHLBaseInfoExtractor):
 
 class NHLIE(InfoExtractor):
     IE_NAME = 'nhl.com'
-    _VALID_URL = r'https?://(?:www\.)?nhl\.com/([^/]+/)*c-(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?(?P<site>nhl|wch2016)\.com/(?:[^/]+/)*c-(?P<id>\d+)'
+    _SITES_MAP = {
+        'nhl': 'nhl',
+        'wch2016': 'wch',
+    }
     _TESTS = [{
         # type=video
         'url': 'https://www.nhl.com/video/anisimov-cleans-up-mess/t-277752844/c-43663503',
@@ -270,13 +274,20 @@ class NHLIE(InfoExtractor):
             'upload_date': '20160204',
             'timestamp': 1454544904,
         },
+    }, {
+        'url': 'https://www.wch2016.com/video/caneur-best-of-game-2-micd-up/t-281230378/c-44983703',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.wch2016.com/news/3-stars-team-europe-vs-team-canada/c-282195068',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
-        tmp_id = self._match_id(url)
+        mobj = re.match(self._VALID_URL, url)
+        tmp_id, site = mobj.group('id'), mobj.group('site')
         video_data = self._download_json(
-            'https://nhl.bamcontent.com/nhl/id/v1/%s/details/web-v1.json' % tmp_id,
-            tmp_id)
+            'https://nhl.bamcontent.com/%s/id/v1/%s/details/web-v1.json'
+            % (self._SITES_MAP[site], tmp_id), tmp_id)
         if video_data.get('type') == 'article':
             video_data = video_data['media']
 

youtube_dl/extractor/niconico.py

@@ -1,4 +1,4 @@
-# encoding: utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re

youtube_dl/extractor/noco.py

@@ -1,4 +1,4 @@
-# encoding: utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re

youtube_dl/extractor/normalboots.py

@@ -1,4 +1,4 @@
-# encoding: utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
 from .common import InfoExtractor

youtube_dl/extractor/nova.py

@@ -1,4 +1,4 @@
-# encoding: utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re

youtube_dl/extractor/nowness.py

@@ -1,4 +1,4 @@
-# encoding: utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
 from .brightcove import (

youtube_dl/extractor/npo.py

@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_HTTPError
 from ..utils import (
     fix_xml_ampersands,
     orderedSet,
@@ -10,6 +11,7 @@ from ..utils import (
     qualities,
     strip_jsonp,
     unified_strdate,
+    ExtractorError,
 )
 
 
@@ -181,9 +183,16 @@ class NPOIE(NPOBaseIE):
                     continue
                 streams = format_info.get('streams')
                 if streams:
-                    video_info = self._download_json(
-                        streams[0] + '&type=json',
-                        video_id, 'Downloading %s stream JSON' % format_id)
+                    try:
+                        video_info = self._download_json(
+                            streams[0] + '&type=json',
+                            video_id, 'Downloading %s stream JSON' % format_id)
+                    except ExtractorError as ee:
+                        if isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 404:
+                            error = (self._parse_json(ee.cause.read().decode(), video_id, fatal=False) or {}).get('errorstring')
+                            if error:
+                                raise ExtractorError(error, expected=True)
+                        raise
                 else:
                     video_info = format_info
                 video_url = video_info.get('url')
@@ -459,8 +468,9 @@ class NPOPlaylistBaseIE(NPOIE):
 
 class VPROIE(NPOPlaylistBaseIE):
     IE_NAME = 'vpro'
-    _VALID_URL = r'https?://(?:www\.)?(?:tegenlicht\.)?vpro\.nl/(?:[^/]+/){2,}(?P<id>[^/]+)\.html'
-    _PLAYLIST_TITLE_RE = r'<h1[^>]+class=["\'].*?\bmedia-platform-title\b.*?["\'][^>]*>([^<]+)'
+    _VALID_URL = r'https?://(?:www\.)?(?:(?:tegenlicht\.)?vpro|2doc)\.nl/(?:[^/]+/)*(?P<id>[^/]+)\.html'
+    _PLAYLIST_TITLE_RE = (r'<h1[^>]+class=["\'].*?\bmedia-platform-title\b.*?["\'][^>]*>([^<]+)',
+                          r'<h5[^>]+class=["\'].*?\bmedia-platform-subtitle\b.*?["\'][^>]*>([^<]+)')
     _PLAYLIST_ENTRY_RE = r'data-media-id="([^"]+)"'
 
     _TESTS = [
@@ -492,6 +502,27 @@ class VPROIE(NPOPlaylistBaseIE):
                 'title': 'education education',
             },
             'playlist_count': 2,
+        },
+        {
+            'url': 'http://www.2doc.nl/documentaires/series/2doc/2015/oktober/de-tegenprestatie.html',
+            'info_dict': {
+                'id': 'de-tegenprestatie',
+                'title': 'De Tegenprestatie',
+            },
+            'playlist_count': 2,
+        }, {
+            'url': 'http://www.2doc.nl/speel~VARA_101375237~mh17-het-verdriet-van-nederland~.html',
+            'info_dict': {
+                'id': 'VARA_101375237',
+                'ext': 'm4v',
+                'title': 'MH17: Het verdriet van Nederland',
+                'description': 'md5:09e1a37c1fdb144621e22479691a9f18',
+                'upload_date': '20150716',
+            },
+            'params': {
+                # Skip because of m3u8 download
+                'skip_download': True
+            },
         }
     ]
 

youtube_dl/extractor/nrk.py

@@ -1,4 +1,4 @@
-# encoding: utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re

youtube_dl/extractor/ntvru.py

@@ -1,4 +1,4 @@
-# encoding: utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
 from .common import InfoExtractor

youtube_dl/extractor/nuevo.py

@@ -1,4 +1,4 @@
-# encoding: utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
 from .common import InfoExtractor

youtube_dl/extractor/nzz.py

@@ -0,0 +1,36 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    extract_attributes,
+)
+
+
+class NZZIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?nzz\.ch/(?:[^/]+/)*[^/?#]+-ld\.(?P<id>\d+)'
+    _TEST = {
+        'url': 'http://www.nzz.ch/zuerich/gymizyte/gymizyte-schreiben-schueler-heute-noch-diktate-ld.9153',
+        'info_dict': {
+            'id': '9153',
+        },
+        'playlist_mincount': 6,
+    }
+
+    def _real_extract(self, url):
+        page_id = self._match_id(url)
+        webpage = self._download_webpage(url, page_id)
+
+        entries = []
+        for player_element in re.findall(r'(<[^>]+class="kalturaPlayer"[^>]*>)', webpage):
+            player_params = extract_attributes(player_element)
+            if player_params.get('data-type') not in ('kaltura_singleArticle',):
+                self.report_warning('Unsupported player type')
+                continue
+            entry_id = player_params['data-id']
+            entries.append(self.url_result(
+                'kaltura:1750922:' + entry_id, 'Kaltura', entry_id))
+
+        return self.playlist_result(entries, page_id)

youtube_dl/extractor/oktoberfesttv.py

@@ -1,4 +1,4 @@
-# encoding: utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
 from .common import InfoExtractor

youtube_dl/extractor/pandoratv.py

@@ -1,4 +1,4 @@
-# encoding: utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
 from .common import InfoExtractor

youtube_dl/extractor/patreon.py

@@ -1,4 +1,4 @@
-# encoding: utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
 from .common import InfoExtractor

youtube_dl/extractor/porn91.py

@@ -1,4 +1,4 @@
-# encoding: utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
 from ..compat import (

youtube_dl/extractor/pornoxo.py

@@ -2,13 +2,13 @@ from __future__ import unicode_literals
 
 import re
 
-from .common import InfoExtractor
+from .jwplatform import JWPlatformBaseIE
 from ..utils import (
     str_to_int,
 )
 
 
-class PornoXOIE(InfoExtractor):
+class PornoXOIE(JWPlatformBaseIE):
     _VALID_URL = r'https?://(?:www\.)?pornoxo\.com/videos/(?P<id>\d+)/(?P<display_id>[^/]+)\.html'
     _TEST = {
         'url': 'http://www.pornoxo.com/videos/7564/striptease-from-sexy-secretary.html',
@@ -17,7 +17,8 @@ class PornoXOIE(InfoExtractor):
             'id': '7564',
             'ext': 'flv',
             'title': 'Striptease From Sexy Secretary!',
-            'description': 'Striptease From Sexy Secretary!',
+            'display_id': 'striptease-from-sexy-secretary',
+            'description': 'md5:0ee35252b685b3883f4a1d38332f9980',
             'categories': list,  # NSFW
             'thumbnail': 're:https?://.*\.jpg$',
             'age_limit': 18,
@@ -26,23 +27,14 @@ class PornoXOIE(InfoExtractor):
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id, display_id = mobj.groups()
 
         webpage = self._download_webpage(url, video_id)
-
-        video_url = self._html_search_regex(
-            r'\'file\'\s*:\s*"([^"]+)"', webpage, 'video_url')
+        video_data = self._extract_jwplayer_data(webpage, video_id, require_title=False)
 
         title = self._html_search_regex(
             r'<title>([^<]+)\s*-\s*PornoXO', webpage, 'title')
 
-        description = self._html_search_regex(
-            r'<meta name="description" content="([^"]+)\s*featuring',
-            webpage, 'description', fatal=False)
-
-        thumbnail = self._html_search_regex(
-            r'\'image\'\s*:\s*"([^"]+)"', webpage, 'thumbnail', fatal=False)
-
         view_count = str_to_int(self._html_search_regex(
             r'[vV]iews:\s*([0-9,]+)', webpage, 'view count', fatal=False))
 
@@ -53,13 +45,14 @@ class PornoXOIE(InfoExtractor):
             None if categories_str is None
             else categories_str.split(','))
 
-        return {
+        video_data.update({
             'id': video_id,
-            'url': video_url,
             'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
+            'display_id': display_id,
+            'description': self._html_search_meta('description', webpage),
             'categories': categories,
             'view_count': view_count,
             'age_limit': 18,
-        }
+        })
+
+        return video_data

youtube_dl/extractor/prosiebensat1.py

@@ -1,4 +1,4 @@
-# encoding: utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re

youtube_dl/extractor/puls4.py

@@ -1,4 +1,4 @@
-# -*- coding: utf-8 -*-
+# coding: utf-8
 from __future__ import unicode_literals
 
 from .prosiebensat1 import ProSiebenSat1BaseIE

youtube_dl/extractor/radiobremen.py

@@ -1,4 +1,4 @@
-# -*- coding: utf-8 -*-
+# coding: utf-8
 
 from __future__ import unicode_literals
 

youtube_dl/extractor/rmcdecouverte.py

@@ -1,4 +1,4 @@
-# encoding: utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
 from .common import InfoExtractor

youtube_dl/extractor/rtl2.py

@@ -1,4 +1,4 @@
-# encoding: utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re

youtube_dl/extractor/rtve.py

@@ -1,4 +1,4 @@
-# encoding: utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
 import base64

youtube_dl/extractor/ruhd.py

@@ -1,4 +1,4 @@
-# -*- coding: utf-8 -*-
+# coding: utf-8
 from __future__ import unicode_literals
 
 from .common import InfoExtractor

youtube_dl/extractor/rutube.py

@@ -1,4 +1,4 @@
-# encoding: utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re

youtube_dl/extractor/rutv.py

@@ -1,4 +1,4 @@
-# encoding: utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re

youtube_dl/extractor/safari.py

@@ -1,4 +1,4 @@
-# encoding: utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re

youtube_dl/extractor/sapo.py

@@ -1,4 +1,4 @@
-# encoding: utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re

youtube_dl/extractor/sbs.py

@@ -1,4 +1,4 @@
-# -*- coding: utf-8 -*-
+# coding: utf-8
 from __future__ import unicode_literals
 
 from .common import InfoExtractor

youtube_dl/extractor/screencast.py

@@ -1,4 +1,4 @@
-# -*- coding: utf-8 -*-
+# coding: utf-8
 from __future__ import unicode_literals
 
 from .common import InfoExtractor

youtube_dl/extractor/screenwavemedia.py

@@ -1,4 +1,4 @@
-# encoding: utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re

youtube_dl/extractor/smotri.py

@@ -1,4 +1,4 @@
-# encoding: utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re

youtube_dl/extractor/sohu.py

@@ -1,4 +1,4 @@
-# encoding: utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re

youtube_dl/extractor/soundcloud.py

@@ -1,4 +1,4 @@
-# encoding: utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re

youtube_dl/extractor/southpark.py

@@ -1,4 +1,4 @@
-# encoding: utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
 from .mtv import MTVServicesInfoExtractor

youtube_dl/extractor/spiegel.py

@@ -1,4 +1,4 @@
-# encoding: utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re

youtube_dl/extractor/srmediathek.py

@@ -1,4 +1,4 @@
-# encoding: utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
 from .ard import ARDMediathekIE

youtube_dl/extractor/streamcz.py

@@ -1,4 +1,4 @@
-# -*- coding: utf-8 -*-
+# coding: utf-8
 from __future__ import unicode_literals
 
 import hashlib

youtube_dl/extractor/swrmediathek.py

@@ -1,4 +1,4 @@
-# -*- coding: utf-8 -*-
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re

youtube_dl/extractor/sztvhu.py

@@ -1,4 +1,4 @@
-# -*- coding: utf-8 -*-
+# coding: utf-8
 from __future__ import unicode_literals
 
 from .common import InfoExtractor

youtube_dl/extractor/tagesschau.py

@@ -1,4 +1,4 @@
-# -*- coding: utf-8 -*-
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re

youtube_dl/extractor/tass.py

@@ -1,4 +1,4 @@
-# encoding: utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
 import json

youtube_dl/extractor/teachertube.py

@@ -1,4 +1,4 @@
-# -*- coding: utf-8 -*-
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re

youtube_dl/extractor/teamcoco.py

@@ -1,4 +1,4 @@
-# -*- coding: utf-8 -*-
+# coding: utf-8
 from __future__ import unicode_literals
 
 import base64

youtube_dl/extractor/techtalks.py

@@ -10,9 +10,9 @@ from ..utils import (
 
 
 class TechTalksIE(InfoExtractor):
-    _VALID_URL = r'https?://techtalks\.tv/talks/[^/]*/(?P<id>\d+)/'
+    _VALID_URL = r'https?://techtalks\.tv/talks/(?:[^/]+/)?(?P<id>\d+)'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://techtalks.tv/talks/learning-topic-models-going-beyond-svd/57758/',
         'info_dict': {
             'id': '57758',
@@ -38,7 +38,10 @@ class TechTalksIE(InfoExtractor):
             # rtmp download
             'skip_download': True,
         },
-    }
+    }, {
+        'url': 'http://techtalks.tv/talks/57758',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)

youtube_dl/extractor/theintercept.py

@@ -1,4 +1,4 @@
-# encoding: utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
 from .common import InfoExtractor

youtube_dl/extractor/theplatform.py

@@ -1,4 +1,4 @@
-# -*- coding: utf-8 -*-
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re

youtube_dl/extractor/theweatherchannel.py

@@ -0,0 +1,79 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .theplatform import ThePlatformIE
+from ..utils import (
+    determine_ext,
+    parse_duration,
+)
+
+
+class TheWeatherChannelIE(ThePlatformIE):
+    _VALID_URL = r'https?://(?:www\.)?weather\.com/(?:[^/]+/)*video/(?P<id>[^/?#]+)'
+    _TESTS = [{
+        'url': 'https://weather.com/series/great-outdoors/video/ice-climber-is-in-for-a-shock',
+        'md5': 'ab924ac9574e79689c24c6b95e957def',
+        'info_dict': {
+            'id': 'cc82397e-cc3f-4d11-9390-a785add090e8',
+            'ext': 'mp4',
+            'title': 'Ice Climber Is In For A Shock',
+            'description': 'md5:55606ce1378d4c72e6545e160c9d9695',
+            'uploader': 'TWC - Digital (No Distro)',
+            'uploader_id': '6ccd5455-16bb-46f2-9c57-ff858bb9f62c',
+        }
+    }]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        drupal_settings = self._parse_json(self._search_regex(
+            r'jQuery\.extend\(Drupal\.settings\s*,\s*({.+?})\);',
+            webpage, 'drupal settings'), display_id)
+        video_id = drupal_settings['twc']['contexts']['node']['uuid']
+        video_data = self._download_json(
+            'https://dsx.weather.com/cms/v4/asset-collection/en_US/' + video_id, video_id)
+        seo_meta = video_data.get('seometa', {})
+        title = video_data.get('title') or seo_meta['title']
+
+        urls = []
+        thumbnails = []
+        formats = []
+        for variant_id, variant_url in video_data.get('variants', []).items():
+            variant_url = variant_url.strip()
+            if not variant_url or variant_url in urls:
+                continue
+            urls.append(variant_url)
+            ext = determine_ext(variant_url)
+            if ext == 'jpg':
+                thumbnails.append({
+                    'url': variant_url,
+                    'id': variant_id,
+                })
+            elif ThePlatformIE.suitable(variant_url):
+                tp_formats, _ = self._extract_theplatform_smil(variant_url, video_id)
+                formats.extend(tp_formats)
+            elif ext == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    variant_url, video_id, 'mp4', 'm3u8_native',
+                    m3u8_id=variant_id, fatal=False))
+            elif ext == 'f4m':
+                formats.extend(self._extract_f4m_formats(
+                    variant_url, video_id, f4m_id=variant_id, fatal=False))
+            else:
+                formats.append({
+                    'url': variant_url,
+                    'format_id': variant_id,
+                })
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'title': title,
+            'description': video_data.get('description') or seo_meta.get('description') or seo_meta.get('og:description'),
+            'duration': parse_duration(video_data.get('duration')),
+            'uploader': video_data.get('providername'),
+            'uploader_id': video_data.get('providerid'),
+            'thumbnails': thumbnails,
+            'formats': formats,
+        }

youtube_dl/extractor/thisoldhouse.py

@@ -0,0 +1,32 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class ThisOldHouseIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?thisoldhouse\.com/(?:watch|how-to)/(?P<id>[^/?#]+)'
+    _TESTS = [{
+        'url': 'https://www.thisoldhouse.com/how-to/how-to-build-storage-bench',
+        'md5': '568acf9ca25a639f0c4ff905826b662f',
+        'info_dict': {
+            'id': '2REGtUDQ',
+            'ext': 'mp4',
+            'title': 'How to Build a Storage Bench',
+            'description': 'In the workshop, Tom Silva and Kevin O\'Connor build a storage bench for an entryway.',
+            'timestamp': 1442548800,
+            'upload_date': '20150918',
+        }
+    }, {
+        'url': 'https://www.thisoldhouse.com/watch/arlington-arts-crafts-arts-and-crafts-class-begins',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        drupal_settings = self._parse_json(self._search_regex(
+            r'jQuery\.extend\(Drupal\.settings\s*,\s*({.+?})\);',
+            webpage, 'drupal settings'), display_id)
+        video_id = drupal_settings['jwplatform']['video_id']
+        return self.url_result('jwplatform:' + video_id, 'JWPlatform', video_id)

youtube_dl/extractor/tlc.py

@@ -1,4 +1,4 @@
-# encoding: utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re

youtube_dl/extractor/tonline.py

@@ -0,0 +1,59 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import int_or_none
+
+
+class TOnlineIE(InfoExtractor):
+    IE_NAME = 't-online.de'
+    _VALID_URL = r'https?://(?:www\.)?t-online\.de/tv/(?:[^/]+/)*id_(?P<id>\d+)'
+    _TEST = {
+        'url': 'http://www.t-online.de/tv/sport/fussball/id_79166266/drittes-remis-zidane-es-muss-etwas-passieren-.html',
+        'md5': '7d94dbdde5f9d77c5accc73c39632c29',
+        'info_dict': {
+            'id': '79166266',
+            'ext': 'mp4',
+            'title': 'Drittes Remis! Zidane: "Es muss etwas passieren"',
+            'description': 'Es läuft nicht rund bei Real Madrid. Das 1:1 gegen den SD Eibar war das dritte Unentschieden in Folge in der Liga.',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        video_data = self._download_json(
+            'http://www.t-online.de/tv/id_%s/tid_json_video' % video_id, video_id)
+        title = video_data['subtitle']
+
+        formats = []
+        for asset in video_data.get('assets', []):
+            asset_source = asset.get('source') or asset.get('source2')
+            if not asset_source:
+                continue
+            formats_id = []
+            for field_key in ('type', 'profile'):
+                field_value = asset.get(field_key)
+                if field_value:
+                    formats_id.append(field_value)
+            formats.append({
+                'format_id': '-'.join(formats_id),
+                'url': asset_source,
+            })
+
+        thumbnails = []
+        for image in video_data.get('images', []):
+            image_source = image.get('source')
+            if not image_source:
+                continue
+            thumbnails.append({
+                'url': image_source,
+            })
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': video_data.get('description'),
+            'duration': int_or_none(video_data.get('duration')),
+            'thumbnails': thumbnails,
+            'formats': formats,
+        }

youtube_dl/extractor/toypics.py

@@ -1,4 +1,4 @@
-# -*- coding:utf-8 -*-
+# coding: utf-8
 from __future__ import unicode_literals
 
 from .common import InfoExtractor