release 2013.02.02

Makefile

@@ -7,6 +7,7 @@ PREFIX=/usr/local
 BINDIR=$(PREFIX)/bin
 MANDIR=$(PREFIX)/man
 SYSCONFDIR=/etc
+PYTHON=/usr/bin/env python
 
 install: youtube-dl youtube-dl.1 youtube-dl.bash-completion
 	install -d $(DESTDIR)$(BINDIR)
@@ -27,7 +28,7 @@ tar: youtube-dl.tar.gz
 youtube-dl: youtube_dl/*.py
 	zip --quiet youtube-dl youtube_dl/*.py
 	zip --quiet --junk-paths youtube-dl youtube_dl/__main__.py
-	echo '#!/usr/bin/env python' > youtube-dl
+	echo '#!$(PYTHON)' > youtube-dl
 	cat youtube-dl.zip >> youtube-dl
 	rm youtube-dl.zip
 	chmod a+x youtube-dl

README.md

@@ -38,6 +38,10 @@ which means you can modify it, redistribute it or use it however you like.
     --reject-title REGEX     skip download for matching titles (regex or
                              caseless sub-string)
     --max-downloads NUMBER   Abort after downloading NUMBER files
+    --min-filesize SIZE      Do not download any videos smaller than SIZE (e.g.
+                             50k or 44.6m)
+    --max-filesize SIZE      Do not download any videos larger than SIZE (e.g.
+                             50k or 44.6m)
 
 ## Filesystem Options:
     -t, --title              use title in file name

devscripts/release.sh

@@ -83,4 +83,7 @@ ROOT=$(pwd)
 )
 rm -rf build
 
+echo "Uploading to PyPi ..."
+pip sdist upload
+
 echo "\n### DONE!"

setup.py

@@ -2,10 +2,14 @@
 # -*- coding: utf-8 -*-
 
 from __future__ import print_function
-from distutils.core import setup
 import pkg_resources
 import sys
 
+try:
+    from setuptools import setup
+except ImportError:
+    from distutils.core import setup
+
 try:
     import py2exe
     """This will create an exe that needs Microsoft Visual C++ 2008 Redistributable Package"""

test/test_download.py

@@ -98,7 +98,7 @@ def generator(test_case):
 
             for tc in test_cases:
                 if not test_case.get('params', {}).get('skip_download', False):
-                    self.assertTrue(os.path.exists(tc['file']))
+                    self.assertTrue(os.path.exists(tc['file']), msg='Missing file ' + tc['file'])
                     self.assertTrue(tc['file'] in finished_hook_called)
                 self.assertTrue(os.path.exists(tc['file'] + '.info.json'))
                 if 'md5' in tc:

test/tests.json

@@ -76,7 +76,8 @@
     "name": "StanfordOpenClassroom",
     "md5":  "544a9468546059d4e80d76265b0443b8",
     "url":  "http://openclassroom.stanford.edu/MainFolder/VideoPage.php?course=PracticalUnix&video=intro-environment&speed=100",
-    "file":  "PracticalUnix_intro-environment.mp4"
+    "file":  "PracticalUnix_intro-environment.mp4",
+    "skip": "Currently offline"
   },
   {
     "name": "XNXX",
@@ -113,8 +114,7 @@
     "name": "Escapist",
     "url": "http://www.escapistmagazine.com/videos/view/the-escapist-presents/6618-Breaking-Down-Baldurs-Gate",
     "file": "6618-Breaking-Down-Baldurs-Gate.flv",
-    "md5": "c6793dbda81388f4264c1ba18684a74d",
-    "skip": "Fails with timeout on Travis"
+    "md5": "c6793dbda81388f4264c1ba18684a74d"
   },
   {
     "name": "GooglePlus",
@@ -181,37 +181,12 @@
   },
   {
     "name": "ComedyCentral",
-    "url": "http://www.thedailyshow.com/full-episodes/thu-december-13-2012-kristen-stewart",
-    "playlist": [
-      {
-        "file": "422204.mp4",
-        "md5": "7a7abe068b31ff03e7b8a37596e72380",
-        "info_dict": {
-            "title": "thedailyshow-thu-december-13-2012-kristen-stewart part 1"
-        }
-      },
-      {
-        "file": "422205.mp4",
-        "md5": "30552b7274c94dbb933f64600eadddd2",
-        "info_dict": {
-            "title": "thedailyshow-thu-december-13-2012-kristen-stewart part 2"
-        }
-      },
-      {
-        "file": "422206.mp4",
-        "md5": "1f4c0664b352cb8e8fe85d5da4fbee91",
-        "info_dict": {
-            "title": "thedailyshow-thu-december-13-2012-kristen-stewart part 3"
-        }
-      },
-      {
-        "file": "422207.mp4",
-        "md5": "f61ee8a4e6bd1308438e03badad78554",
-        "info_dict": {
-            "title": "thedailyshow-thu-december-13-2012-kristen-stewart part 4"
-        }
-      }
-    ]
+    "url": "http://www.thedailyshow.com/watch/thu-december-13-2012/kristen-stewart",
+    "file": "422212.mp4",
+    "md5": "4e2f5cb088a83cd8cdb7756132f9739d",
+    "info_dict": {
+        "title": "thedailyshow-kristen-stewart part 1"
+    }
   },
   {
     "name": "RBMARadio",
@@ -244,56 +219,59 @@
         "file": "11885610.m4a",
         "md5": "96ce57f24389fc8734ce47f4c1abcc55",
         "info_dict": {
-          "title": "youtube-dl test track 1 \"'/\\\u00e4\u21ad"
+          "title": "youtue-dl project<>\"' - youtube-dl test track 1 \"'/\\\u00e4\u21ad",
+          "uploader_id": "ytdl"
         }
       },
       {
         "file": "11885608.m4a",
         "md5": "4ab26f05c1f7291ea460a3920be8021f",
         "info_dict": {
-          "title": "youtube-dl test track 2 \"'/\\\u00e4\u21ad"
+          "title": "youtube-dl project - youtube-dl test track 2 \"'/\\\u00e4\u21ad",
+          "uploader_id": "ytdl"
+
         }
       },
       {
         "file": "11885679.m4a",
         "md5": "d30b5b5f74217410f4689605c35d1fd7",
         "info_dict": {
-          "title": "youtube-dl test track 3 \"'/\\\u00e4\u21ad"
+          "title": "youtube-dl project as well - youtube-dl test track 3 \"'/\\\u00e4\u21ad"
         }
       },
       {
         "file": "11885680.m4a",
         "md5": "4eb0a669317cd725f6bbd336a29f923a",
         "info_dict": {
-          "title": "youtube-dl test track 4 \"'/\\\u00e4\u21ad"
+          "title": "youtube-dl project as well - youtube-dl test track 4 \"'/\\\u00e4\u21ad"
         }
       },
       {
         "file": "11885682.m4a",
         "md5": "1893e872e263a2705558d1d319ad19e8",
         "info_dict": {
-          "title": "youtube-dl test track 5 \"'/\\\u00e4\u21ad"
+          "title": "PH - youtube-dl test track 5 \"'/\\\u00e4\u21ad"
         }
       },
       {
         "file": "11885683.m4a",
         "md5": "b673c46f47a216ab1741ae8836af5899",
         "info_dict": {
-          "title": "youtube-dl test track 6 \"'/\\\u00e4\u21ad"
+          "title": "PH - youtube-dl test track 6 \"'/\\\u00e4\u21ad"
         }
       },
       {
         "file": "11885684.m4a",
         "md5": "1d74534e95df54986da7f5abf7d842b7",
         "info_dict": {
-          "title": "youtube-dl test track 7 \"'/\\\u00e4\u21ad"
+          "title": "phihag - youtube-dl test track 7 \"'/\\\u00e4\u21ad"
         }
       },
       {
         "file": "11885685.m4a",
         "md5": "f081f47af8f6ae782ed131d38b9cd1c0",
         "info_dict": {
-          "title": "youtube-dl test track 8 \"'/\\\u00e4\u21ad"
+          "title": "phihag - youtube-dl test track 8 \"'/\\\u00e4\u21ad"
         }
       }
     ]

youtube_dl/FileDownloader.py

@@ -82,6 +82,8 @@ class FileDownloader(object):
     subtitleslang:     Language of the subtitles to download
     test:              Download only first bytes to test the downloader.
     keepvideo:         Keep the video file after post-processing
+    min_filesize:      Skip files smaller than this size
+    max_filesize:      Skip files larger than this size
     """
 
     params = None
@@ -712,6 +714,15 @@ class FileDownloader(object):
         data_len = data.info().get('Content-length', None)
         if data_len is not None:
             data_len = int(data_len) + resume_len
+            min_data_len = self.params.get("min_filesize", None)
+            max_data_len =  self.params.get("max_filesize", None)
+            if min_data_len is not None and data_len < min_data_len:
+                self.to_screen(u'\r[download] File is smaller than min-filesize (%s bytes < %s bytes). Aborting.' % (data_len, min_data_len))
+                return False
+            if max_data_len is not None and data_len > max_data_len:
+                self.to_screen(u'\r[download] File is larger than max-filesize (%s bytes > %s bytes). Aborting.' % (data_len, max_data_len))
+                return False
+
         data_len_str = self.format_bytes(data_len)
         byte_counter = 0 + resume_len
         block_size = self.params.get('buffersize', 1024)

youtube_dl/InfoExtractors.py

@@ -2050,8 +2050,10 @@ class FacebookIE(InfoExtractor):
         if not m:
             raise ExtractorError(u'Cannot parse data')
         data = dict(json.loads(m.group(1)))
-        video_url = compat_urllib_parse.unquote(data['hd_src'])
-        video_duration = int(data['video_duration'])
+        params_raw = compat_urllib_parse.unquote(data['params'])
+        params = json.loads(params_raw)
+        video_url = params['hd_src']
+        video_duration = int(params['video_duration'])
 
         m = re.search('<h2 class="uiHeaderTitle">([^<]+)</h2>', webpage)
         if not m:
@@ -2064,7 +2066,7 @@ class FacebookIE(InfoExtractor):
             'url': video_url,
             'ext': 'mp4',
             'duration': video_duration,
-            'thumbnail': data['thumbnail_src'],
+            'thumbnail': params['thumbnail_src'],
         }
         return [info]
 
@@ -2869,8 +2871,7 @@ class StanfordOpenClassroomIE(InfoExtractor):
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         if mobj is None:
-            self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
-            return
+            raise ExtractorError(u'Invalid URL: %s' % url)
 
         if mobj.group('course') and mobj.group('video'): # A specific video
             course = mobj.group('course')
@@ -2907,12 +2908,9 @@ class StanfordOpenClassroomIE(InfoExtractor):
                 'upload_date': None,
             }
 
-            self.report_download_webpage(info['id'])
-            try:
-                coursepage = compat_urllib_request.urlopen(url).read()
-            except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
-                self._downloader.trouble(u'ERROR: unable to download course info page: ' + compat_str(err))
-                return
+            coursepage = self._download_webpage(url, info['id'],
+                                        note='Downloading course info page',
+                                        errnote='Unable to download course info page')
 
             m = re.search('<h1>([^<]+)</h1>', coursepage)
             if m:
@@ -2936,7 +2934,6 @@ class StanfordOpenClassroomIE(InfoExtractor):
                 assert entry['type'] == 'reference'
                 results += self.extract(entry['url'])
             return results
-
         else: # Root page
             info = {
                 'id': 'Stanford OpenClassroom',
@@ -3861,7 +3858,7 @@ class YouJizzIE(InfoExtractor):
 
 class EightTracksIE(InfoExtractor):
     IE_NAME = '8tracks'
-    _VALID_URL = r'https?://8tracks.com/(?P<user>[^/]+)/(?P<id>[^/]+)'
+    _VALID_URL = r'https?://8tracks.com/(?P<user>[^/]+)/(?P<id>[^/#]+)(?:#.*)?$'
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
@@ -3892,7 +3889,9 @@ class EightTracksIE(InfoExtractor):
             info = {
                 'id': track_data['id'],
                 'url': track_data['track_file_stream_url'],
-                'title': track_data['name'],
+                'title': track_data['performer'] + u' - ' + track_data['name'],
+                'raw_title': track_data['name'],
+                'uploader_id': data['user']['login'],
                 'ext': 'm4a',
             }
             res.append(info)

youtube_dl/PostProcessor.py

@@ -143,10 +143,10 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
 
         more_opts = []
         if self._preferredcodec == 'best' or self._preferredcodec == filecodec or (self._preferredcodec == 'm4a' and filecodec == 'aac'):
-            if self._preferredcodec == 'm4a' and filecodec == 'aac':
+            if filecodec == 'aac' and self._preferredcodec in ['m4a', 'best']:
                 # Lossless, but in another container
                 acodec = 'copy'
-                extension = self._preferredcodec
+                extension = 'm4a'
                 more_opts = [self._exes['avconv'] and '-bsf:a' or '-absf', 'aac_adtstoasc']
             elif filecodec in ['aac', 'mp3', 'vorbis', 'opus']:
                 # Lossless if possible

youtube_dl/__init__.py

@@ -150,6 +150,9 @@ def parseOpts():
     selection.add_option('--match-title', dest='matchtitle', metavar='REGEX',help='download only matching titles (regex or caseless sub-string)')
     selection.add_option('--reject-title', dest='rejecttitle', metavar='REGEX',help='skip download for matching titles (regex or caseless sub-string)')
     selection.add_option('--max-downloads', metavar='NUMBER', dest='max_downloads', help='Abort after downloading NUMBER files', default=None)
+    selection.add_option('--min-filesize', metavar='SIZE', dest='min_filesize', help="Do not download any videos smaller than SIZE (e.g. 50k or 44.6m)", default=None)
+    selection.add_option('--max-filesize', metavar='SIZE', dest='max_filesize', help="Do not download any videos larger than SIZE (e.g. 50k or 44.6m)", default=None)
+
 
     authentication.add_option('-u', '--username',
             dest='username', metavar='USERNAME', help='account username')
@@ -286,10 +289,13 @@ def _real_main():
     else:
         try:
             jar = compat_cookiejar.MozillaCookieJar(opts.cookiefile)
-            if os.path.isfile(opts.cookiefile) and os.access(opts.cookiefile, os.R_OK):
+            if os.access(opts.cookiefile, os.R_OK):
                 jar.load()
         except (IOError, OSError) as err:
-            sys.exit(u'ERROR: unable to open cookie file')
+            if opts.verbose:
+                traceback.print_exc()
+            sys.stderr.write(u'ERROR: unable to open cookie file\n')
+            sys.exit(101)
     # Set user agent
     if opts.user_agent is not None:
         std_headers['User-Agent'] = opts.user_agent
@@ -349,6 +355,16 @@ def _real_main():
         if numeric_limit is None:
             parser.error(u'invalid rate limit specified')
         opts.ratelimit = numeric_limit
+    if opts.min_filesize is not None:
+        numeric_limit = FileDownloader.parse_bytes(opts.min_filesize)
+        if numeric_limit is None:
+            parser.error(u'invalid min_filesize specified')
+        opts.min_filesize = numeric_limit
+    if opts.max_filesize is not None:
+        numeric_limit = FileDownloader.parse_bytes(opts.max_filesize)
+        if numeric_limit is None:
+            parser.error(u'invalid max_filesize specified')
+        opts.max_filesize = numeric_limit
     if opts.retries is not None:
         try:
             opts.retries = int(opts.retries)
@@ -438,6 +454,8 @@ def _real_main():
         'verbose': opts.verbose,
         'test': opts.test,
         'keepvideo': opts.keepvideo,
+        'min_filesize': opts.min_filesize,
+        'max_filesize': opts.max_filesize
         })
 
     if opts.verbose:

youtube_dl/utils.py

@@ -280,6 +280,12 @@ class AttrParser(compat_html_parser.HTMLParser):
             lines[-1] = lines[-1][:self.result[2][1]-self.result[1][1]]
         lines[-1] = lines[-1][:self.result[2][1]]
         return '\n'.join(lines).strip()
+# Hack for https://github.com/rg3/youtube-dl/issues/662
+if sys.version_info < (2, 7, 3):
+    AttrParser.parse_endtag = (lambda self, i:
+        i + len("</scr'+'ipt>")
+        if self.rawdata[i:].startswith("</scr'+'ipt>")
+        else compat_html_parser.HTMLParser.parse_endtag(self, i))
 
 def get_element_by_id(id, html):
     """Return the content of the tag with the specified ID in the passed HTML document"""

youtube_dl/version.py

@@ -1,2 +1,2 @@
 
-__version__ = '2013.01.27'
+__version__ = '2013.02.02'