From b6541100a38f394cdfdba4818c2a45144bf04936 Mon Sep 17 00:00:00 2001 From: Olof Johansson Date: Tue, 29 Mar 2016 19:49:54 +0200 Subject: [PATCH 1/8] select_quality: fix argument parsing Instead of parsing the argument to --stream-prio as a comma separated listed, it was accidentally handled as a space separated list. --- lib/svtplay_dl/utils/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/svtplay_dl/utils/__init__.py b/lib/svtplay_dl/utils/__init__.py index 1e272df..6f0c7c6 100644 --- a/lib/svtplay_dl/utils/__init__.py +++ b/lib/svtplay_dl/utils/__init__.py @@ -117,7 +117,9 @@ def select_quality(options, streams): # Extract protocol prio, in the form of "hls,hds,http,rtmp", # we want it as a list - proto_prio = (options.stream_prio or '').split() or None + proto_prio = None + if options.stream_prio: + proto_prio = options.stream_prio.split(',') return [x for x in prio_streams(streams, protocol_prio=proto_prio) From a6e05e45026f32594f5fc16ddbb0469be166617b Mon Sep 17 00:00:00 2001 From: Olof Johansson Date: Tue, 29 Mar 2016 19:39:26 +0200 Subject: [PATCH 2/8] prio_streams: make protocol_prio param mandatory Move the responsibility for extracting it to select_quality (prio_streams' caller). This makes the prio_streams function simpler. And at the same time, move the default protocol_prio list to global scope. This can for instance be used for improved error reporting. --- lib/svtplay_dl/tests/prio_streams.py | 15 ++------------- lib/svtplay_dl/utils/__init__.py | 12 +++++++----- 2 files changed, 9 insertions(+), 18 deletions(-) diff --git a/lib/svtplay_dl/tests/prio_streams.py b/lib/svtplay_dl/tests/prio_streams.py index 71140b1..aecf5d6 100644 --- a/lib/svtplay_dl/tests/prio_streams.py +++ b/lib/svtplay_dl/tests/prio_streams.py @@ -16,44 +16,33 @@ class Stream(object): return '%s(%d)' % (self.proto.upper(), self.bitrate) class PrioStreamsTest(unittest.TestCase): - def _gen_proto_case(self, ordered, unordered, default=True, expected=None): + def _gen_proto_case(self, ordered, unordered, expected=None): streams = [Stream(x, 100) for x in unordered] kwargs = {} - if not default: - kwargs['protocol_prio'] = ordered if expected is None: expected = [str(Stream(x, 100)) for x in ordered] return self.assertEqual( - [str(x) for x in prio_streams(streams, **kwargs)], + [str(x) for x in prio_streams(streams, ordered, **kwargs)], expected ) - def test_default_order(self): - return self._gen_proto_case( - ['hls', 'hds', 'http', 'rtmp'], - ['rtmp', 'hds', 'hls', 'http'] - ) - def test_custom_order(self): return self._gen_proto_case( ['http', 'rtmp', 'hds', 'hls'], ['rtmp', 'hds', 'hls', 'http'], - default=False, ) def test_custom_order_1(self): return self._gen_proto_case( ['http'], ['rtmp', 'hds', 'hls', 'http'], - default=False, ) def test_proto_unavail(self): return self._gen_proto_case( ['http', 'rtmp'], ['hds', 'hls', 'https'], - default=False, expected=[], ) diff --git a/lib/svtplay_dl/utils/__init__.py b/lib/svtplay_dl/utils/__init__.py index 6f0c7c6..9ead5ec 100644 --- a/lib/svtplay_dl/utils/__init__.py +++ b/lib/svtplay_dl/utils/__init__.py @@ -26,6 +26,9 @@ is_py2_old = (sys.version_info < (2, 7)) # Used for UA spoofing in get_http_data() FIREFOX_UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.3' +# TODO: should be set as the default option in the argument parsing? +DEFAULT_PROTOCOL_PRIO = ["dash", "hls", "hds", "http", "rtmp"] + log = logging.getLogger('svtplay_dl') progress_stream = sys.stderr @@ -69,12 +72,11 @@ def list_quality(videos): for i in data: log.info("%s\t%s" % (i[0], i[1].upper())) -def prio_streams(streams, protocol_prio=None): - if protocol_prio is None: - protocol_prio = ["dash", "hls", "hds", "http", "rtmp"] - +def prio_streams(streams, protocol_prio): # Map score's to the reverse of the list's index values proto_score = dict(zip(protocol_prio, range(len(protocol_prio), 0, -1))) + log.debug("Protocol priority scores (higher is better): %s", + str(proto_score)) # Build a tuple (bitrate, proto_score, stream), and use it # for sorting. @@ -117,7 +119,7 @@ def select_quality(options, streams): # Extract protocol prio, in the form of "hls,hds,http,rtmp", # we want it as a list - proto_prio = None + proto_prio = DEFAULT_PROTOCOL_PRIO if options.stream_prio: proto_prio = options.stream_prio.split(',') From 51c71aa1cbe84a28fd31a467ad8dfd2e26bb63a6 Mon Sep 17 00:00:00 2001 From: Olof Johansson Date: Tue, 29 Mar 2016 19:42:46 +0200 Subject: [PATCH 3/8] error: New exception, NoRequestedProtocols This excpetion is thrown when the stream can't be accessed by any accepted protocol (as decided by options.stream_prio). --- lib/svtplay_dl/error.py | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/lib/svtplay_dl/error.py b/lib/svtplay_dl/error.py index 0fa0a87..dd6d4a1 100644 --- a/lib/svtplay_dl/error.py +++ b/lib/svtplay_dl/error.py @@ -7,4 +7,32 @@ class UIException(Exception): pass class ServiceError(Exception): - pass \ No newline at end of file + pass + +class NoRequestedProtocols(UIException): + """ + This excpetion is thrown when the service provides streams, + but not using any accepted protocol (as decided by + options.stream_prio). + """ + + def __init__(self, requested, found): + """ + The constructor takes two mandatory parameters, requested + and found. Both should be lists. requested is the protocols + we want and found is the protocols that can be used to + access the stream. + """ + self.requested = requested + self.found = found + + super(NoRequestedProtocols, self).__init__( + "None of the provided protocols (%s) are in " + "the current list of accepted protocols (%s)" % ( + self.found, self.requested + ) + ) + + def __repr__(self): + return "NoRequestedProtocols(requested=%s, found=%s)" % ( + self.requested, self.found) From 9d2054b4bc730ade9b00f13b5b9d3b0efe52a042 Mon Sep 17 00:00:00 2001 From: Olof Johansson Date: Tue, 29 Mar 2016 20:01:24 +0200 Subject: [PATCH 4/8] select_quality: handle when no requested proto is available Needs to widen the scope of the try: catch block in svtplay_dl/__init__.py a little, since select_quality can now also fire away UIExceptions. --- lib/svtplay_dl/__init__.py | 12 ++++++------ lib/svtplay_dl/utils/__init__.py | 16 ++++++++++++---- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/lib/svtplay_dl/__init__.py b/lib/svtplay_dl/__init__.py index 5983409..9a51556 100644 --- a/lib/svtplay_dl/__init__.py +++ b/lib/svtplay_dl/__init__.py @@ -231,13 +231,13 @@ def get_one_media(stream, options): if options.list_quality: list_quality(videos) return - stream = select_quality(options, videos) - log.info("Selected to download %s, bitrate: %s", - stream.name(), stream.bitrate) - if options.get_url: - print(stream.url) - return try: + stream = select_quality(options, videos) + log.info("Selected to download %s, bitrate: %s", + stream.name(), stream.bitrate) + if options.get_url: + print(stream.url) + return stream.download() except UIException as e: if options.verbose: diff --git a/lib/svtplay_dl/utils/__init__.py b/lib/svtplay_dl/utils/__init__.py index 9ead5ec..44b8d2a 100644 --- a/lib/svtplay_dl/utils/__init__.py +++ b/lib/svtplay_dl/utils/__init__.py @@ -19,6 +19,8 @@ except ImportError: print("You need to install python-requests to use this script") sys.exit(3) +from svtplay_dl import error + is_py2 = (sys.version_info[0] == 2) is_py3 = (sys.version_info[0] == 3) is_py2_old = (sys.version_info < (2, 7)) @@ -123,9 +125,15 @@ def select_quality(options, streams): if options.stream_prio: proto_prio = options.stream_prio.split(',') - return [x for - x in prio_streams(streams, protocol_prio=proto_prio) - if x.bitrate == selected][0] + try: + return [x for + x in prio_streams(streams, protocol_prio=proto_prio) + if x.bitrate == selected][0] + except IndexError: + raise error.NoRequestedProtocols( + requested=proto_prio, + found=list(set([s.name() for s in streams])) + ) def ensure_unicode(s): @@ -212,4 +220,4 @@ def which(program): exe_file = os.path.join(os.getcwd(), program) if is_exe(exe_file): return exe_file - return None \ No newline at end of file + return None From dad2790d9e10133263ab236b52413d41b959eb65 Mon Sep 17 00:00:00 2001 From: Olof Johansson Date: Tue, 29 Mar 2016 20:46:48 +0200 Subject: [PATCH 5/8] select_quality: Filter out unwanted protocols before bitrate It was easily possible to end up in a state where the bitrate prioritization wanted a bitrate only avaiable via protocols outside of our set of accepted protocols, like trying to disable dash for svtplay. By doing the protocol filtering first, we end up only considering "valid" bitrates. --- lib/svtplay_dl/utils/__init__.py | 34 ++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/lib/svtplay_dl/utils/__init__.py b/lib/svtplay_dl/utils/__init__.py index 44b8d2a..b346d69 100644 --- a/lib/svtplay_dl/utils/__init__.py +++ b/lib/svtplay_dl/utils/__init__.py @@ -88,6 +88,22 @@ def prio_streams(streams, protocol_prio): x in sorted(prioritized, key=itemgetter(0,1), reverse=True)] def select_quality(options, streams): + # Extract protocol prio, in the form of "hls,hds,http,rtmp", + # we want it as a list + proto_prio = DEFAULT_PROTOCOL_PRIO + if options.stream_prio: + proto_prio = options.stream_prio.split(',') + + # Filter away any unwanted protocols, and prioritize + # based on --stream-priority. + streams = prio_streams(streams, proto_prio) + + if len(streams) == 0: + raise error.NoRequestedProtocols( + requested=proto_prio, + found=list(set([s.name() for s in streams])) + ) + available = sorted(int(x.bitrate) for x in streams) try: optq = int(options.quality) @@ -119,21 +135,9 @@ def select_quality(options, streams): sys.exit(4) - # Extract protocol prio, in the form of "hls,hds,http,rtmp", - # we want it as a list - proto_prio = DEFAULT_PROTOCOL_PRIO - if options.stream_prio: - proto_prio = options.stream_prio.split(',') - - try: - return [x for - x in prio_streams(streams, protocol_prio=proto_prio) - if x.bitrate == selected][0] - except IndexError: - raise error.NoRequestedProtocols( - requested=proto_prio, - found=list(set([s.name() for s in streams])) - ) + for s in streams: + if s.bitrate == selected: + return s def ensure_unicode(s): From 84ca17a14c00d19dbc645f4f83043341f01b34ac Mon Sep 17 00:00:00 2001 From: Olof Johansson Date: Tue, 29 Mar 2016 21:30:42 +0200 Subject: [PATCH 6/8] select_quality: Replace sys.exits with UIExceptions --- lib/svtplay_dl/utils/__init__.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/lib/svtplay_dl/utils/__init__.py b/lib/svtplay_dl/utils/__init__.py index b346d69..e4ae3f8 100644 --- a/lib/svtplay_dl/utils/__init__.py +++ b/lib/svtplay_dl/utils/__init__.py @@ -108,14 +108,12 @@ def select_quality(options, streams): try: optq = int(options.quality) except ValueError: - log.error("Requested quality need to be a number") - sys.exit(4) + raise error.UIException("Requested quality needs to be a number") if optq: try: optf = int(options.flexibleq) except ValueError: - log.error("Flexible-quality need to be a number") - sys.exit(4) + raise error.UIException("Flexible-quality needs to be a number") if not optf: wanted = [optq] else: @@ -131,9 +129,8 @@ def select_quality(options, streams): if not selected and selected != 0: data = sort_quality(streams) quality = ", ".join("%s (%s)" % (str(x), str(y)) for x, y in data) - log.error("Can't find that quality. Try one of: %s (or try --flexible-quality)", quality) - - sys.exit(4) + raise error.UIException("Can't find that quality. Try one of: %s (or " + "try --flexible-quality)" % quality) for s in streams: if s.bitrate == selected: From f59207302d051918583b841c73ad1ba40a4aba43 Mon Sep 17 00:00:00 2001 From: Olof Johansson Date: Wed, 30 Mar 2016 23:09:13 +0200 Subject: [PATCH 7/8] prio_streams: Rename to protocol_prio --- .../tests/{prio_streams.py => protocol_prio.py} | 4 ++-- lib/svtplay_dl/utils/__init__.py | 12 +++++++++--- 2 files changed, 11 insertions(+), 5 deletions(-) rename lib/svtplay_dl/tests/{prio_streams.py => protocol_prio.py} (91%) diff --git a/lib/svtplay_dl/tests/prio_streams.py b/lib/svtplay_dl/tests/protocol_prio.py similarity index 91% rename from lib/svtplay_dl/tests/prio_streams.py rename to lib/svtplay_dl/tests/protocol_prio.py index aecf5d6..df0a2d7 100644 --- a/lib/svtplay_dl/tests/prio_streams.py +++ b/lib/svtplay_dl/tests/protocol_prio.py @@ -4,7 +4,7 @@ from __future__ import absolute_import import unittest -from svtplay_dl.utils import prio_streams +from svtplay_dl.utils import protocol_prio class Stream(object): def __init__(self, proto, bitrate): @@ -24,7 +24,7 @@ class PrioStreamsTest(unittest.TestCase): expected = [str(Stream(x, 100)) for x in ordered] return self.assertEqual( - [str(x) for x in prio_streams(streams, ordered, **kwargs)], + [str(x) for x in protocol_prio(streams, ordered, **kwargs)], expected ) diff --git a/lib/svtplay_dl/utils/__init__.py b/lib/svtplay_dl/utils/__init__.py index e4ae3f8..8c6e9fb 100644 --- a/lib/svtplay_dl/utils/__init__.py +++ b/lib/svtplay_dl/utils/__init__.py @@ -74,9 +74,15 @@ def list_quality(videos): for i in data: log.info("%s\t%s" % (i[0], i[1].upper())) -def prio_streams(streams, protocol_prio): +def protocol_prio(streams, priolist): + """ + Given a list of VideoRetriever objects and a prioritized list of + accepted protocols (as strings) (highest priority first), return + a list of VideoRetriever objects that are accepted, and sorted + by bitrate, and then protocol priority. + """ # Map score's to the reverse of the list's index values - proto_score = dict(zip(protocol_prio, range(len(protocol_prio), 0, -1))) + proto_score = dict(zip(priolist, range(len(priolist), 0, -1))) log.debug("Protocol priority scores (higher is better): %s", str(proto_score)) @@ -96,7 +102,7 @@ def select_quality(options, streams): # Filter away any unwanted protocols, and prioritize # based on --stream-priority. - streams = prio_streams(streams, proto_prio) + streams = protocol_prio(streams, proto_prio) if len(streams) == 0: raise error.NoRequestedProtocols( From fa66beff9b36c9011c4313363a84356c7ebc6fa9 Mon Sep 17 00:00:00 2001 From: Olof Johansson Date: Tue, 29 Mar 2016 21:36:02 +0200 Subject: [PATCH 8/8] select_quality: Simplify and add comments --- lib/svtplay_dl/utils/__init__.py | 58 ++++++++++++++++++-------------- 1 file changed, 32 insertions(+), 26 deletions(-) diff --git a/lib/svtplay_dl/utils/__init__.py b/lib/svtplay_dl/utils/__init__.py index 8c6e9fb..cb03ed5 100644 --- a/lib/svtplay_dl/utils/__init__.py +++ b/lib/svtplay_dl/utils/__init__.py @@ -94,6 +94,16 @@ def protocol_prio(streams, priolist): x in sorted(prioritized, key=itemgetter(0,1), reverse=True)] def select_quality(options, streams): + try: + optq = int(options.quality) + except ValueError: + raise error.UIException("Requested quality needs to be a number") + + try: + optf = int(options.flexibleq) + except ValueError: + raise error.UIException("Flexible-quality needs to be a number") + # Extract protocol prio, in the form of "hls,hds,http,rtmp", # we want it as a list proto_prio = DEFAULT_PROTOCOL_PRIO @@ -110,38 +120,34 @@ def select_quality(options, streams): found=list(set([s.name() for s in streams])) ) - available = sorted(int(x.bitrate) for x in streams) - try: - optq = int(options.quality) - except ValueError: - raise error.UIException("Requested quality needs to be a number") - if optq: - try: - optf = int(options.flexibleq) - except ValueError: - raise error.UIException("Flexible-quality needs to be a number") - if not optf: - wanted = [optq] - else: - wanted = range(optq-optf, optq+optf+1) - else: - wanted = [available[-1]] + # Build a dict indexed by bitrate, where each value + # is the stream with the highest priority protocol. + stream_hash = {} + for s in streams: + if not s.bitrate in stream_hash: + stream_hash[s.bitrate] = s - selected = None - for q in available: - if q in wanted: - selected = q - break - if not selected and selected != 0: + avail = sorted(stream_hash.keys(), reverse=True) + + # wanted_lim is a two element tuple defines lower/upper bounds + # (inclusive). By default, we want only the best for you + # (literally!). + wanted_lim = (avail[0],)*2 + if optq: + wanted_lim = (optq - optf, optq + optf) + + # wanted is the filtered list of available streams, having + # a bandwidth within the wanted_lim range. + wanted = [a for a in avail if a >= wanted_lim[0] and a <= wanted_lim[1]] + + # If none remains, the bitrate filtering was too tight. + if len(wanted) == 0: data = sort_quality(streams) quality = ", ".join("%s (%s)" % (str(x), str(y)) for x, y in data) raise error.UIException("Can't find that quality. Try one of: %s (or " "try --flexible-quality)" % quality) - for s in streams: - if s.bitrate == selected: - return s - + return stream_hash[wanted[0]] def ensure_unicode(s): """