From 6e5a1af449428a7ac99ae24e2ff972a014fc6976 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20Wikstr=C3=B6m?= Date: Fri, 18 Jan 2013 10:31:13 +0100 Subject: [PATCH 01/50] svtplay: don't crach when using --hls --- svtplay_dl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/svtplay_dl.py b/svtplay_dl.py index ba098b9..417a9e5 100755 --- a/svtplay_dl.py +++ b/svtplay_dl.py @@ -987,7 +987,7 @@ class Svtplay(): streams2[int(i["bitrate"])] = stream if len(streams) == 0 and options.hls: - test = streams2[list(streams.keys())[0]] + test = streams2[0] test["url"] = test["url"].replace("/z/", "/i/").replace("manifest.f4m", "master.m3u8") elif len(streams) == 0: log.error("Can't find any streams.") From cc14a26c60e010cd6a9e2adabbbe06812a01ecb9 Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Fri, 18 Jan 2013 10:44:06 +0100 Subject: [PATCH 02/50] Update the version number --- svtplay_dl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/svtplay_dl.py b/svtplay_dl.py index 417a9e5..4459c09 100755 --- a/svtplay_dl.py +++ b/svtplay_dl.py @@ -25,7 +25,7 @@ import struct import binascii from datetime import timedelta -__version__ = "0.8.2013.01.15" +__version__ = "0.8.2013.01.18" class Options: """ From 2aaf4ac8a6b8f57cdef85a9daa26748c17b52495 Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Sat, 26 Jan 2013 20:50:22 +0100 Subject: [PATCH 03/50] aftonbladet: get only take 3 arguments. --- svtplay_dl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/svtplay_dl.py b/svtplay_dl.py index 4459c09..0cff87a 100755 --- a/svtplay_dl.py +++ b/svtplay_dl.py @@ -834,7 +834,7 @@ class Aftonbladet(): def handle(self, url): return "aftonbladet.se" in url - def get(self, options, url, start): + def get(self, options, url): parse = urlparse(url) data = get_http_data(url) match = re.search("abTvArticlePlayer-player-(.*)-[0-9]+-[0-9]+-clickOverlay", data) From b82937e18944e9b1e4c1d7fd73af09d31ae7507a Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Sat, 26 Jan 2013 20:51:44 +0100 Subject: [PATCH 04/50] aftonbladet: handle live streams automagic --- svtplay_dl.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/svtplay_dl.py b/svtplay_dl.py index 0cff87a..75de3dc 100755 --- a/svtplay_dl.py +++ b/svtplay_dl.py @@ -25,7 +25,7 @@ import struct import binascii from datetime import timedelta -__version__ = "0.8.2013.01.18" +__version__ = "0.8.2013.01.26" class Options: """ @@ -850,11 +850,15 @@ class Aftonbladet(): xml = ET.XML(data) url = xml.find("articleElement").find("mediaElement").find("baseUrl").text path = xml.find("articleElement").find("mediaElement").find("media").attrib["url"] + live = xml.find("articleElement").find("mediaElement").find("isLive").text options.other = "-y %s" % path if start > 0: options.other = "%s -A %s" % (options.other, str(start)) + if live == "true": + options.live = True + if url == None: log.error("Can't find any video on that page") sys.exit(3) From 98b0fa57a173901dcd89a03ac2527107210554ae Mon Sep 17 00:00:00 2001 From: Patrik Ragnarsson Date: Mon, 28 Jan 2013 21:50:10 +0100 Subject: [PATCH 05/50] Switch to Markdown for README --- README => README.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename README => README.md (100%) diff --git a/README b/README.md similarity index 100% rename from README rename to README.md From 135d84a25e8cd21749280cdc3849cf15f12e027a Mon Sep 17 00:00:00 2001 From: Patrik Ragnarsson Date: Mon, 28 Jan 2013 22:01:13 +0100 Subject: [PATCH 06/50] Update README.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Links to dependencies. Mention the Homebrew formula. --- README.md | 61 ++++++++++++++++++++++++++++++++----------------------- 1 file changed, 36 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 891288a..e5a9b47 100644 --- a/README.md +++ b/README.md @@ -1,26 +1,37 @@ -You need to install rtmpdump to make it work -and you need python-pyamf for kanal5play +svtplay-dl +========== -this script works for: -aftonbladet.se -di.se -dn.se -dr.dk -expressen.se -hbo.com -justin.tv -kanal5.se -kanal5play.se -kanal9play.se -nrk.no -ruv.is -svd.se -sverigesradio.se -svtplay.se -tv4.se -tv3play.se -tv4play.se -tv6play.se -tv8play.se -twitch.tv -urplay.se +You need: + +* [RTMPDump](http://rtmpdump.mplayerhq.hu/) 2.4 or higher +* [PyCrypto](https://www.dlitz.net/software/pycrypto/) to download encrypted HLS streams +* [PyAMF](http://www.pyamf.org/) for kanal5play.se + +This script works for: + +* aftonbladet.se +* di.se +* dn.se +* dr.dk +* expressen.se +* hbo.com +* justin.tv +* kanal5.se +* kanal5play.se +* kanal9play.se +* nrk.no +* ruv.is +* svd.se +* sverigesradio.se +* svtplay.se +* tv4.se +* tv3play.se +* tv4play.se +* tv6play.se +* tv8play.se +* twitch.tv +* urplay.se + +If you have OS X and [Homebrew](http://mxcl.github.com/homebrew/) you can install with: + + brew install svtplay-dl From b7b50b101f6926d84b6a4321fd3dc7adf64320e9 Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Mon, 28 Jan 2013 22:07:39 +0100 Subject: [PATCH 07/50] Update the comment about pyamf --- README.md | 2 +- svtplay_dl.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e5a9b47..b27a3e5 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ You need: * [RTMPDump](http://rtmpdump.mplayerhq.hu/) 2.4 or higher * [PyCrypto](https://www.dlitz.net/software/pycrypto/) to download encrypted HLS streams -* [PyAMF](http://www.pyamf.org/) for kanal5play.se +* [PyAMF](http://www.pyamf.org/) for kanal5.se / kanal9play.se This script works for: diff --git a/svtplay_dl.py b/svtplay_dl.py index 75de3dc..631f662 100755 --- a/svtplay_dl.py +++ b/svtplay_dl.py @@ -769,7 +769,7 @@ class Kanal9(): try: from pyamf import remoting except ImportError: - log.error("You need to install pyamf to download content from kanal5 and kanal9") + log.error("You need to install pyamf to download content from kanal5.se and kanal9play") log.error("In debian the package is called python-pyamf") sys.exit(2) From 8e547f436d141c6de18af3cf5fd39bf51c739e10 Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Mon, 28 Jan 2013 22:47:54 +0100 Subject: [PATCH 08/50] tv4play: some fixes for tv4.se In some cases we dont have vid in the url --- svtplay_dl.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/svtplay_dl.py b/svtplay_dl.py index 631f662..aa948fd 100755 --- a/svtplay_dl.py +++ b/svtplay_dl.py @@ -910,8 +910,13 @@ class Tv4play(): if match: vid = match.group(1) else: - log.error("Can't find video file") - sys.exit(2) + data = get_http_data(url) + match = re.search("\"vid\":\"(\d+)\",", data) + if match: + vid = match.group(1) + else: + log.error("Can't find video file") + sys.exit(2) url = "http://premium.tv4play.se/api/web/asset/%s/play" % vid data = get_http_data(url) From c42c664128768f227df50f4eb19a9627132c9262 Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Fri, 8 Feb 2013 12:22:45 +0100 Subject: [PATCH 09/50] download_http: fix a problem when content-length is not present. --- svtplay_dl.py | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/svtplay_dl.py b/svtplay_dl.py index aa948fd..1f49bc6 100755 --- a/svtplay_dl.py +++ b/svtplay_dl.py @@ -307,21 +307,25 @@ def get_http_data(url, method="GET", header="", data=""): def progress(byte, total, extra = ""): """ Print some info about how much we have downloaded """ - ratio = float(byte) / total - percent = round(ratio*100, 2) - tlen = str(len(str(total))) - fmt = "Downloaded %"+tlen+"dkB of %dkB bytes (% 3.2f%%)" - progresstr = fmt % (byte >> 10, total >> 10, percent) + if total == 0: + progresstr = "Downloaded %dkB bytes" % (byte >> 10) + progress_stream.write(progresstr + '\r') + else: + ratio = float(byte) / total + percent = round(ratio*100, 2) + tlen = str(len(str(total))) + fmt = "Downloaded %"+tlen+"dkB of %dkB bytes (% 3.2f%%)" + progresstr = fmt % (byte >> 10, total >> 10, percent) - columns = int(os.getenv("COLUMNS", "80")) - if len(progresstr) < columns - 13: - p = int((columns - len(progresstr) - 3) * ratio) - q = int((columns - len(progresstr) - 3) * (1 - ratio)) - progresstr = "[" + ("#" * p) + (" " * q) + "] " + progresstr - progress_stream.write(progresstr + ' ' + extra + '\r') + columns = int(os.getenv("COLUMNS", "80")) + if len(progresstr) < columns - 13: + p = int((columns - len(progresstr) - 3) * ratio) + q = int((columns - len(progresstr) - 3) * (1 - ratio)) + progresstr = "[" + ("#" * p) + (" " * q) + "] " + progresstr + progress_stream.write(progresstr + ' ' + extra + '\r') - if byte >= total: - progress_stream.write('\n') + if byte >= total: + progress_stream.write('\n') progress_stream.flush() @@ -464,7 +468,10 @@ def download_hls(options, url, baseurl=None): def download_http(options, url): """ Get the stream from HTTP """ response = urlopen(url) - total_size = response.info()['Content-Length'] + try: + total_size = response.info()['Content-Length'] + except KeyError: + total_size = 0 total_size = int(total_size) bytes_so_far = 0 if options.output != "-": From 289cd22b45e4635564f4549ffbf7f7e74641754a Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Fri, 8 Feb 2013 12:34:28 +0100 Subject: [PATCH 10/50] Support for radioplay.se --- svtplay_dl.py | 44 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/svtplay_dl.py b/svtplay_dl.py index 1f49bc6..02f3b71 100755 --- a/svtplay_dl.py +++ b/svtplay_dl.py @@ -1078,6 +1078,46 @@ class Ruv(object): base_url = m3u8_url.rsplit("/", 1)[0] download_hls(options, m3u8_url, base_url) +class Radioplay(object): + def handle(self, url): + return "radioplay.se" in url + + def get(self, options, url): + data = get_http_data(url) + match = re.search("liveStationsRedundancy = ({.*});", data) + parse = urlparse(url) + station = parse.path[1:] + streams = None + if match: + data = json.loads(match.group(1)) + for i in data["stations"]: + if station == i["name"].lower().replace(" ", ""): + streams = i["streams"] + break + else: + log.error("Can't find any streams.") + sys.exit(2) + if streams: + if options.hls: + try: + m3u8_url = streams["hls"] + base_url = m3u8_url.rsplit("/", 1)[0] + download_hls(options, m3u8_url, base_url) + except KeyError: + log.error("Can't find any streams.") + sys.error(2) + else: + try: + rtmp = streams["rtmp"] + download_rtmp(options, rtmp) + except KeyError: + mp3 = streams["mp3"] + download_http(options, mp3) + + else: + log.error("Can't find any streams.") + sys.exit(2) + def progressbar(total, pos, msg=""): """ Given a total and a progress position, output a progress bar @@ -1113,8 +1153,8 @@ def progressbar(total, pos, msg=""): progress_stream.write(fmt % (pos, total, bar, msg)) def get_media(url, options): - sites = [Aftonbladet(), Dr(), Expressen(), Hbo(), Justin(), Kanal5(), Kanal9(), - Nrk(), Qbrick(), Ruv(), Sr(), Svtplay(), Tv4play(), Urplay(), Viaplay()] + sites = [Aftonbladet(), Dr(), Expressen(), Hbo(), Justin(), Kanal5(), Kanal9(), Nrk(), + Qbrick(), Ruv(), Radioplay(), Sr(), Svtplay(), Tv4play(), Urplay(), Viaplay()] stream = None for i in sites: if i.handle(url): From db85d3ede7c832939c32e86ea6e97e0102fb09ed Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Fri, 8 Feb 2013 12:37:43 +0100 Subject: [PATCH 11/50] Generic class a way to find embeded videos support for embeded svtplay videos. --- svtplay_dl.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/svtplay_dl.py b/svtplay_dl.py index 02f3b71..5343df6 100755 --- a/svtplay_dl.py +++ b/svtplay_dl.py @@ -1118,6 +1118,20 @@ class Radioplay(object): log.error("Can't find any streams.") sys.exit(2) +class generic(object): + ''' Videos embed in sites ''' + def get(self, sites, url): + data = get_http_data(url) + match = re.search("src=\"(http://www.svt.se/wd.*)\" frameborder", data) + stream = None + if match: + url = match.group(1) + for i in sites: + if i.handle(url): + stream = i + break + return url, stream + def progressbar(total, pos, msg=""): """ Given a total and a progress position, output a progress bar @@ -1160,10 +1174,13 @@ def get_media(url, options): if i.handle(url): stream = i break - if not stream: - log.error("That site is not supported. Make a ticket or send a message") - sys.exit(2) + if not stream: + url, stream = generic().get(sites, url) + if not stream: + log.error("That site is not supported. Make a ticket or send a message") + sys.exit(2) + url = url.replace("&", "&") if not options.output or os.path.isdir(options.output): data = get_http_data(url) match = re.search("(?i)\s*(.*?)\s*", data) From 3007584c39d024becf6e357d1344a35ef7f5959b Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Thu, 14 Feb 2013 23:42:30 +0100 Subject: [PATCH 12/50] urplay: add support for the new site some help from @unayok --- svtplay_dl.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/svtplay_dl.py b/svtplay_dl.py index 5343df6..bf9712d 100755 --- a/svtplay_dl.py +++ b/svtplay_dl.py @@ -663,11 +663,22 @@ class Urplay(): def get(self, options, url): data = get_http_data(url) - match = re.search('file=(.*)\&plugins', data) - if match: - path = "mp%s:%s" % (match.group(1)[-1], match.group(1)) - options.other = "-a ondemand -y %s" % path - download_rtmp(options, "rtmp://streaming.ur.se/") + data = re.search("urPlayer.init\((.*)\);", data) + data = re.sub("(\w+): ", r'"\1":',data.group(1)) + data = data.replace("\'", "\"").replace("\",}","\"}").replace("(m = location.hash.match(/[#&]start=(\d+)/)) ? m[1] : 0,","0") + jsondata = json.loads(data) + basedomain = jsondata["streaming_config"]["streamer"]["redirect"] + http = "http://%s/%s" % (basedomain, jsondata["file_html5"]) + hds = "%s%s" % (http, jsondata["streaming_config"]["http_streaming"]["hds_file"]) + hls = "%s%s" % (http, jsondata["streaming_config"]["http_streaming"]["hls_file"]) + rtmp = "rtmp://%s/%s" % (basedomain, jsondata["streaming_config"]["rtmp"]["application"]) + path = "mp%s:%s" % (jsondata["file_flash"][-1], jsondata["file_flash"]) + options.other = "-v -a %s -y %s" % (jsondata["streaming_config"]["rtmp"]["application"], path) + if options.hls: + download_hls(options, hls, http) + if jsondata["file_flash"][-1] == "3": + download_rtmp(options, rtmp) + download_hds(options, hds) class Qbrick(): def handle(self, url): From a3159938ebf6335a2f8081c482ddc4f2bca67b6a Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Thu, 14 Feb 2013 23:44:53 +0100 Subject: [PATCH 13/50] urplay: adding support for ur.se --- README.md | 1 + svtplay_dl.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index b27a3e5..5fd1339 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,7 @@ This script works for: * tv6play.se * tv8play.se * twitch.tv +* ur.se * urplay.se If you have OS X and [Homebrew](http://mxcl.github.com/homebrew/) you can install with: diff --git a/svtplay_dl.py b/svtplay_dl.py index bf9712d..8ee27f0 100755 --- a/svtplay_dl.py +++ b/svtplay_dl.py @@ -659,7 +659,7 @@ class Sr(): class Urplay(): def handle(self, url): - return "urplay.se" in url + return ("urplay.se" in url) or ("ur.se" in url) def get(self, options, url): data = get_http_data(url) From d206c0c38cf295150fd86256ff8f4109368cdcdd Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Thu, 14 Feb 2013 23:47:00 +0100 Subject: [PATCH 14/50] download_hls: add support for relative urls --- svtplay_dl.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/svtplay_dl.py b/svtplay_dl.py index 8ee27f0..b8484a4 100755 --- a/svtplay_dl.py +++ b/svtplay_dl.py @@ -25,7 +25,7 @@ import struct import binascii from datetime import timedelta -__version__ = "0.8.2013.01.26" +__version__ = "0.8.2013.02.14" class Options: """ @@ -401,6 +401,8 @@ def download_hls(options, url, baseurl=None): streams[int(i[1]["BANDWIDTH"])] = i[0] test = select_quality(options, streams) + if test[1:4] != "http": + test = "%s%s" % (baseurl, test) m3u8 = get_http_data(test) globaldata, files = parsem3u(m3u8) encrypted = False From 7b43bb8420814d7b0e9ba452f9e097900125c0da Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Fri, 15 Feb 2013 10:41:17 +0100 Subject: [PATCH 15/50] urplay: use rtmp instead of hds --- svtplay_dl.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/svtplay_dl.py b/svtplay_dl.py index b8484a4..294e779 100755 --- a/svtplay_dl.py +++ b/svtplay_dl.py @@ -678,9 +678,8 @@ class Urplay(): options.other = "-v -a %s -y %s" % (jsondata["streaming_config"]["rtmp"]["application"], path) if options.hls: download_hls(options, hls, http) - if jsondata["file_flash"][-1] == "3": + else: download_rtmp(options, rtmp) - download_hds(options, hds) class Qbrick(): def handle(self, url): From 0a6bbfba870799ca47087fbe394f46c1cb6073cb Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Fri, 15 Feb 2013 10:42:18 +0100 Subject: [PATCH 16/50] download_rtmp: add a dot before extention and after filename --- svtplay_dl.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/svtplay_dl.py b/svtplay_dl.py index 294e779..b7a1430 100755 --- a/svtplay_dl.py +++ b/svtplay_dl.py @@ -25,7 +25,7 @@ import struct import binascii from datetime import timedelta -__version__ = "0.8.2013.02.14" +__version__ = "0.8.2013.02.15" class Options: """ @@ -519,7 +519,7 @@ def download_rtmp(options, url): if not extension: options.output = "%s.flv" % options.output else: - options.output = "%s%s" % (options.output, extension.group(1)) + options.output = "%s.%s" % (options.output, extension.group(1)) else: options.output = options.output + extension.group(1) log.info("Outfile: %s", options.output) From fa498cf4eaecc720b83d386e02d97bd820f38df4 Mon Sep 17 00:00:00 2001 From: Olof Johansson Date: Fri, 15 Feb 2013 19:58:23 +0100 Subject: [PATCH 17/50] download_hls: Don't assume baseurl is set --- svtplay_dl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/svtplay_dl.py b/svtplay_dl.py index b7a1430..b053927 100755 --- a/svtplay_dl.py +++ b/svtplay_dl.py @@ -401,7 +401,7 @@ def download_hls(options, url, baseurl=None): streams[int(i[1]["BANDWIDTH"])] = i[0] test = select_quality(options, streams) - if test[1:4] != "http": + if baseurl and test[1:4] != "http": test = "%s%s" % (baseurl, test) m3u8 = get_http_data(test) globaldata, files = parsem3u(m3u8) From f7fa14287119102fba059206049daee86de10d35 Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Sat, 16 Feb 2013 18:22:51 +0100 Subject: [PATCH 18/50] download_hds: dont hardcode metadata size. --- svtplay_dl.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/svtplay_dl.py b/svtplay_dl.py index b053927..a6a390a 100755 --- a/svtplay_dl.py +++ b/svtplay_dl.py @@ -368,7 +368,10 @@ def download_hds(options, url, swf=None): else: file_d = sys.stdout - file_d.write(binascii.a2b_hex(b"464c56010500000009000000001200010c00000000000000")) + metasize = struct.pack(">L", len(base64.b64decode(test["metadata"])))[1:] + file_d.write(binascii.a2b_hex(b"464c560105000000090000000012")) + file_d.write(metasize) + file_d.write(binascii.a2b_hex(b"00000000000000")) file_d.write(base64.b64decode(test["metadata"])) file_d.write(binascii.a2b_hex(b"00000000")) total = antal[1]["total"] From 0ddacb8bcca70964cf069bbb00622b8d91dd9bff Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Sat, 16 Feb 2013 21:05:08 +0100 Subject: [PATCH 19/50] download_hds: fix some decoding problems Saw some issues while downloading streams from urplay. --- svtplay_dl.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/svtplay_dl.py b/svtplay_dl.py index a6a390a..4a7172f 100755 --- a/svtplay_dl.py +++ b/svtplay_dl.py @@ -262,10 +262,9 @@ def parsem3u(data): def decode_f4f(fragID, fragData): start = fragData.find("mdat") + 4 if (fragID > 1): - for dummy in range(2): - tagLen, = struct.unpack_from(">L", fragData, start) - tagLen &= 0x00ffffff - start += tagLen + 11 + 4 + tagLen, = struct.unpack_from(">L", fragData, start) + tagLen &= 0x00ffffff + start += tagLen + 11 + 4 return start def get_http_data(url, method="GET", header="", data=""): From 948c39fce349aba299348ab6e9c4b6e7312b6f2b Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Sat, 16 Feb 2013 21:54:09 +0100 Subject: [PATCH 20/50] download_rtmp: work around for a bug in python2.7 python2.7.2 in OSX Mountain Lion has bug in shlex.split(). see http://bugs.python.org/issue6988 --- svtplay_dl.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/svtplay_dl.py b/svtplay_dl.py index 4a7172f..0357c13 100755 --- a/svtplay_dl.py +++ b/svtplay_dl.py @@ -529,7 +529,10 @@ def download_rtmp(options, url): if options.silent or options.output == "-": args.append("-q") if options.other: - args += shlex.split(options.other) + if sys.version_info < (3, 0): + args += shlex.split(options.other.encode("utf-8")) + else: + args += shlex.split(options.other) command = ["rtmpdump", "-r", url] + args try: subprocess.call(command) From 81d17a1bf1b17d9980c54ed41bb94fd687e999dd Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Wed, 20 Feb 2013 19:17:41 +0100 Subject: [PATCH 21/50] Adding few subtitle formats. json, sami, smi and wsrt --- svtplay_dl.py | 76 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/svtplay_dl.py b/svtplay_dl.py index 0357c13..7d69c8c 100755 --- a/svtplay_dl.py +++ b/svtplay_dl.py @@ -58,6 +58,7 @@ class Options: self.quality = None self.hls = False self.other = None + self.subtitle = False log = logging.getLogger('svtplay_dl') progress_stream = sys.stderr @@ -539,6 +540,78 @@ def download_rtmp(options, url): except OSError as e: log.error("Could not execute rtmpdump: " + e.strerror) +def timestr(seconds): + total = float(seconds) / 1000 + hours = int(total / 3600) + minutes = int(total / 60) + sec = total % 60 + output = "%02d:%02d:%02.02f" % (hours, minutes, sec) + return output.replace(".", ",") + +def subtitle_json(options, url): + data = json.loads(get_http_data(url)) + number = 1 + subs = "" + for i in data: + subs += "%s\n%s --> %s\n" % (number, timestr(int(i["startMillis"])), timestr(int(i["endMillis"]))) + subs += "%s\n\n" % i["text"] + number += 1 + + filename = "%s.srt" % options.output + fd = open(filename, "w") + fd.write(subs) + fd.close() + +def subtitle_sami(options, url): + data = get_http_data(url) + tree = ET.XML(data) + subt = tree.find("Font") + subs = "" + for i in subt.getiterator(): + if i.tag == "Subtitle": + if i.attrib["SpotNumber"] == 1: + subs += "%s\n%s --> %s\n" % (i.attrib["SpotNumber"], i.attrib["TimeIn"], i.attrib["TimeOut"]) + else: + subs += "\n%s\n%s --> %s\n" % (i.attrib["SpotNumber"], i.attrib["TimeIn"], i.attrib["TimeOut"]) + else: + subs += "%s\n" % i.text + filename = "%s.srt" % options.output + fd = open(filename, "w") + fd.write(subs) + fd.close() + +def subtitle_smi(options, url): + data = get_http_data(url) + recomp = re.compile(r'\s+

(.*)
\s+\s+

', re.M|re.I|re.U) + number = 1 + subs = "" + for i in recomp.finditer(data): + subs += "%s\n%s --> %s\n" % (number, timestr(i.group(1)), timestr(i.group(3))) + text = "%s\n\n" % i.group(2) + subs += text.replace("
", "\n") + number += 1 + + filename = "%s.srt" % options.output + fd = open(filename, "w") + fd.write(subs) + fd.close() + +def subtitle_wsrt(options, url): + data = get_http_data(url) + recomp = re.compile("(\d+)\r\n([\d:\.]+ --> [\d:\.]+)?([^\r\n]+)?\r\n([^\r\n]+)\r\n(([^\r\n]*)\r\n)?") + srt = "" + for i in recomp.finditer(data): + sub = "%s\n%s\n%s\n" % (i.group(1), i.group(2).replace(".", ","), i.group(4)) + if len(i.group(6)) > 0: + sub += "%s\n" % i.group(6) + sub += "\n" + sub = re.sub('<[^>]*>', '', sub) + srt += sub + filename = "%s.srt" % options.output + fd = open(filename, "w") + fd.write(srt) + fd.close + def select_quality(options, streams): sort = sorted(streams.keys(), key=int) @@ -1250,6 +1323,9 @@ def main(): metavar="quality", help="Choose what format to download.\nIt will download the best format by default") parser.add_option("-H", "--hls", action="store_true", dest="hls", default=False) + parser.add_option("-S", "--subtitle", + action="store_true", dest="subtitle", default=False, + help="Download subtitle from the site if available.") (options, args) = parser.parse_args() if len(args) != 1: parser.error("incorrect number of arguments") From 3f9eb4d48012ebdb96793338a4c8b4cda3e7a5e1 Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Thu, 21 Feb 2013 15:11:19 +0100 Subject: [PATCH 22/50] Output the name of the subtitle. --- svtplay_dl.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/svtplay_dl.py b/svtplay_dl.py index 7d69c8c..ff59406 100755 --- a/svtplay_dl.py +++ b/svtplay_dl.py @@ -557,7 +557,10 @@ def subtitle_json(options, url): subs += "%s\n\n" % i["text"] number += 1 - filename = "%s.srt" % options.output + filename = re.search("(.*)\.[a-z0-9]{2,3}$", options.output) + if filename: + options.output = "%s.srt" % filename.group(1) + log.info("Subtitle: %s", options.output) fd = open(filename, "w") fd.write(subs) fd.close() @@ -575,7 +578,11 @@ def subtitle_sami(options, url): subs += "\n%s\n%s --> %s\n" % (i.attrib["SpotNumber"], i.attrib["TimeIn"], i.attrib["TimeOut"]) else: subs += "%s\n" % i.text - filename = "%s.srt" % options.output + + filename = re.search("(.*)\.[a-z0-9]{2,3}$", options.output) + if filename: + options.output = "%s.srt" % filename.group(1) + log.info("Subtitle: %s", options.output) fd = open(filename, "w") fd.write(subs) fd.close() @@ -591,7 +598,10 @@ def subtitle_smi(options, url): subs += text.replace("
", "\n") number += 1 - filename = "%s.srt" % options.output + filename = re.search("(.*)\.[a-z0-9]{2,3}$", options.output) + if filename: + options.output = "%s.srt" % filename.group(1) + log.info("Subtitle: %s", options.output) fd = open(filename, "w") fd.write(subs) fd.close() @@ -607,10 +617,13 @@ def subtitle_wsrt(options, url): sub += "\n" sub = re.sub('<[^>]*>', '', sub) srt += sub - filename = "%s.srt" % options.output + filename = re.search("(.*)\.[a-z0-9]{2,3}$", options.output) + if filename: + options.output = "%s.srt" % filename.group(1) + log.info("Subtitle: %s", options.output) fd = open(filename, "w") fd.write(srt) - fd.close + fd.close() def select_quality(options, streams): sort = sorted(streams.keys(), key=int) From 72c0282966f5704b3aa191210e1b90d26c16c346 Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Thu, 21 Feb 2013 15:12:47 +0100 Subject: [PATCH 23/50] svtplay: support for subtitle --- svtplay_dl.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/svtplay_dl.py b/svtplay_dl.py index ff59406..4c27952 100755 --- a/svtplay_dl.py +++ b/svtplay_dl.py @@ -1129,6 +1129,13 @@ class Svtplay(): download_hds(options, manifest, swf) else: download_http(options, test["url"]) + if options.subtitle: + try: + suburl = data["video"]["subtitleReferences"][0]["url"] + except KeyError: + sys.exit(1) + if len(suburl) > 0: + subtitle_wsrt(options, suburl) class Nrk(object): def handle(self, url): From 623b54c057a764bd9b69ac601d9c14f39a07d705 Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Thu, 21 Feb 2013 15:21:35 +0100 Subject: [PATCH 24/50] tv4play: support for subtitle. --- svtplay_dl.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/svtplay_dl.py b/svtplay_dl.py index 4c27952..cbb53a9 100755 --- a/svtplay_dl.py +++ b/svtplay_dl.py @@ -1042,6 +1042,7 @@ class Tv4play(): options.live = True streams = {} + subtitle = False for i in sa: if i.find("mediaFormat").text != "smi": @@ -1049,6 +1050,8 @@ class Tv4play(): stream["uri"] = i.find("base").text stream["path"] = i.find("url").text streams[int(i.find("bitrate").text)] = stream + elif i.find("mediaFormat").text == "smi": + subtitle = i.find("url").text if len(streams) == 1: test = streams[list(streams.keys())[0]] else: @@ -1066,6 +1069,8 @@ class Tv4play(): sys.exit(2) manifest = "%s?hdcore=2.8.0&g=hejsan" % test["path"] download_hds(options, manifest, swf) + if options.subtitle and subtitle: + subtitle_smi(options, subtitle) class Svtplay(): def handle(self, url): From 66bb33243abf4bb6cec80e3a489bc1e2edf6758c Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Thu, 21 Feb 2013 19:09:33 +0100 Subject: [PATCH 25/50] expressen: it works now again. --- svtplay_dl.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/svtplay_dl.py b/svtplay_dl.py index cbb53a9..c8b39ec 100755 --- a/svtplay_dl.py +++ b/svtplay_dl.py @@ -911,13 +911,12 @@ class Expressen(): return "expressen.se" in url def get(self, options, url): - parse = urlparse(url) - match = re.search("/(.*[\/\+].*)/", unquote_plus(parse.path)) + data = get_http_data(url) + match = re.search("xmlUrl: '(http://www.expressen.*)'", data) if not match: log.error("Can't find video file") sys.exit(2) - url = "http://tv.expressen.se/%s/?standAlone=true&output=xml" % quote_plus(match.group(1)) - other = "" + url = match.group(1) data = get_http_data(url) xml = ET.XML(data) ss = xml.find("vurls") @@ -933,7 +932,7 @@ class Expressen(): test = select_quality(options, streams) filename = test - match = re.search("rtmp://([0-9a-z\.]+/[0-9]+/)(.*).flv", filename) + match = re.search("rtmp://([0-9a-z\.]+/[0-9]+/)(.*)", filename) filename = "rtmp://%s" % match.group(1) options.other = "-y %s" % match.group(2) From b036149a00519b6e9edf542963381158efbb238b Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Thu, 21 Feb 2013 19:43:42 +0100 Subject: [PATCH 26/50] Adding support for TimedText subtitles --- svtplay_dl.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/svtplay_dl.py b/svtplay_dl.py index c8b39ec..065fbed 100755 --- a/svtplay_dl.py +++ b/svtplay_dl.py @@ -548,6 +548,40 @@ def timestr(seconds): output = "%02d:%02d:%02.02f" % (hours, minutes, sec) return output.replace(".", ",") +def norm(name): + if name[0] == "{": + uri, tag = name[1:].split("}") + return tag + else: + return name + +def subtitle_tt(options, url): + i = 1 + data = "" + skip = False + fh = get_http_data(url) + tree = ET.parse(fh) + for node in tree.iter(): + tag = norm(node.tag) + if tag == "p": + if skip: + data = data + "\n" + data += '%s\n%s,%s --> %s,%s\n' % (i, node.attrib["begin"][:8], node.attrib["begin"][9:], node.attrib["end"][:8], node.attrib["end"][9:]) + data += '%s\n' % node.text.strip(' \t\n\r') + skip = True + i += 1 + if tag == "br": + if node.tail: + data += '%s\n\n' % node.tail.strip(' \t\n\r') + skip = False + filename = re.search("(.*)\.[a-z0-9]{2,3}$", options.output) + if filename: + options.output = "%s.srt" % filename.group(1) + log.info("Subtitle: %s", options.output) + fd = open(filename, "w") + fd.write(data) + fd.close() + def subtitle_json(options, url): data = json.loads(get_http_data(url)) number = 1 From 2febdd855eb6f5b27bac748882d9f096eee513e2 Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Thu, 21 Feb 2013 19:51:52 +0100 Subject: [PATCH 27/50] urplay: support for subtitles --- svtplay_dl.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/svtplay_dl.py b/svtplay_dl.py index 065fbed..9923a9c 100755 --- a/svtplay_dl.py +++ b/svtplay_dl.py @@ -794,6 +794,7 @@ class Urplay(): data = re.sub("(\w+): ", r'"\1":',data.group(1)) data = data.replace("\'", "\"").replace("\",}","\"}").replace("(m = location.hash.match(/[#&]start=(\d+)/)) ? m[1] : 0,","0") jsondata = json.loads(data) + subtitle = jsondata["subtitles"].split(",")[0] basedomain = jsondata["streaming_config"]["streamer"]["redirect"] http = "http://%s/%s" % (basedomain, jsondata["file_html5"]) hds = "%s%s" % (http, jsondata["streaming_config"]["http_streaming"]["hds_file"]) @@ -805,6 +806,8 @@ class Urplay(): download_hls(options, hls, http) else: download_rtmp(options, rtmp) + if options.subtitle: + subtitle_tt(options, subtitle) class Qbrick(): def handle(self, url): From cd9f94f5eb077c905f2b109788dc90bf9683f8d5 Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Thu, 21 Feb 2013 20:05:13 +0100 Subject: [PATCH 28/50] kanal5play: support for subtitles --- svtplay_dl.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/svtplay_dl.py b/svtplay_dl.py index 9923a9c..b3b4f58 100755 --- a/svtplay_dl.py +++ b/svtplay_dl.py @@ -886,6 +886,8 @@ class Kanal5(): data = json.loads(get_http_data(url)) options.live = data["isLive"] steambaseurl = data["streamBaseUrl"] + if data["hasSubtitle"]: + subtitle = "http://www.kanal5play.se/api/subtitles/%s" % match.group(1) streams = {} for i in data["streams"]: @@ -900,6 +902,8 @@ class Kanal5(): options.output = "%s.%s" % (options.output, match.group(1)) options.other = "-W %s -y %s " % ("http://www.kanal5play.se/flash/StandardPlayer.swf", filename) download_rtmp(options, steambaseurl) + if options.subtitle: + subtitle_json(options, subtitle) class Kanal9(): def handle(self, url): From a682423e96f0d6d28bc61640f760b8fd60f4cf47 Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Thu, 21 Feb 2013 20:08:29 +0100 Subject: [PATCH 29/50] kanal5play: support for kanal9play.se --- svtplay_dl.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/svtplay_dl.py b/svtplay_dl.py index b3b4f58..e82a65b 100755 --- a/svtplay_dl.py +++ b/svtplay_dl.py @@ -875,7 +875,7 @@ class Qbrick(): class Kanal5(): def handle(self, url): - return "kanal5play.se" in url + return ("kanal5play.se" in url) or ("kanal9play.se" in url) def get(self, options, url): match = re.search(".*video/([0-9]+)", url) @@ -907,7 +907,7 @@ class Kanal5(): class Kanal9(): def handle(self, url): - return ("kanal9play.se" in url) or ("kanal5.se" in url) + return "kanal5.se" in url def get(self, options, url): data = get_http_data(url) From 6773531ec35dd86d7b7f80d4d9ec352008cdb6bd Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Thu, 21 Feb 2013 20:16:03 +0100 Subject: [PATCH 30/50] kanal5.se is now "closed" --- svtplay_dl.py | 44 +------------------------------------------- 1 file changed, 1 insertion(+), 43 deletions(-) diff --git a/svtplay_dl.py b/svtplay_dl.py index e82a65b..6895fbc 100755 --- a/svtplay_dl.py +++ b/svtplay_dl.py @@ -905,48 +905,6 @@ class Kanal5(): if options.subtitle: subtitle_json(options, subtitle) -class Kanal9(): - def handle(self, url): - return "kanal5.se" in url - - def get(self, options, url): - data = get_http_data(url) - match = re.search("@videoPlayer\" value=\"(.*)\"", data) - if not match: - match = re.search("videoId=(\d+)&player", data) - if not match: - log.error("Can't find video file") - sys.exit(2) - try: - from pyamf import remoting - except ImportError: - log.error("You need to install pyamf to download content from kanal5.se and kanal9play") - log.error("In debian the package is called python-pyamf") - sys.exit(2) - - player_id = 811317479001 - publisher_id = 22710239001 - const = "9f79dd85c3703b8674de883265d8c9e606360c2e" - env = remoting.Envelope(amfVersion=3) - env.bodies.append(("/1", remoting.Request(target="com.brightcove.player.runtime.PlayerMediaFacade.findMediaById", body=[const, player_id, match.group(1), publisher_id], envelope=env))) - env = str(remoting.encode(env).read()) - url = "http://c.brightcove.com/services/messagebroker/amf?playerKey=AQ~~,AAAABUmivxk~,SnCsFJuhbr0vfwrPJJSL03znlhz-e9bk" - header = "application/x-amf" - data = get_http_data(url, "POST", header, env) - streams = {} - - for i in remoting.decode(data).bodies[0][1].body['renditions']: - stream = {} - stream["uri"] = i["defaultURL"] - streams[i["encodingRate"]] = stream - - test = select_quality(options, streams) - - filename = test["uri"] - match = re.search("(rtmp[e]{0,1}://.*)\&(.*)$", filename) - options.other = "-W %s -y %s " % ("http://admin.brightcove.com/viewer/us1.25.04.01.2011-05-24182704/connection/ExternalConnection_2.swf", match.group(2)) - download_rtmp(options, match.group(1)) - class Expressen(): def handle(self, url): return "expressen.se" in url @@ -1322,7 +1280,7 @@ def progressbar(total, pos, msg=""): progress_stream.write(fmt % (pos, total, bar, msg)) def get_media(url, options): - sites = [Aftonbladet(), Dr(), Expressen(), Hbo(), Justin(), Kanal5(), Kanal9(), Nrk(), + sites = [Aftonbladet(), Dr(), Expressen(), Hbo(), Justin(), Kanal5(), Nrk(), Qbrick(), Ruv(), Radioplay(), Sr(), Svtplay(), Tv4play(), Urplay(), Viaplay()] stream = None for i in sites: From afc920fa72dc940750e33c0ca423a7d373cb68e5 Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Thu, 21 Feb 2013 20:26:12 +0100 Subject: [PATCH 31/50] viaplay: support for subtitles --- svtplay_dl.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/svtplay_dl.py b/svtplay_dl.py index 6895fbc..76a4938 100755 --- a/svtplay_dl.py +++ b/svtplay_dl.py @@ -992,6 +992,7 @@ class Viaplay(): data = get_http_data(url) xml = ET.XML(data) filename = xml.find("Product").find("Videos").find("Video").find("Url").text + subtitle = xml.find("Product").find("SamiFile").text if filename[:4] == "http": data = get_http_data(filename) @@ -1000,6 +1001,8 @@ class Viaplay(): options.other = "-W http://flvplayer.viastream.viasat.tv/play/swf/player110516.swf?rnd=1315434062" download_rtmp(options, filename) + if options.subtitle and subtitle: + subtitle_sami(options, subtitle) class Tv4play(): def handle(self, url): From d8433add3fbe2dd2448b4a370d584b98b004f055 Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Thu, 21 Feb 2013 20:28:37 +0100 Subject: [PATCH 32/50] New version. --- svtplay_dl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/svtplay_dl.py b/svtplay_dl.py index 76a4938..57b848f 100755 --- a/svtplay_dl.py +++ b/svtplay_dl.py @@ -25,7 +25,7 @@ import struct import binascii from datetime import timedelta -__version__ = "0.8.2013.02.15" +__version__ = "0.9.2013.02.21" class Options: """ From 537c0247a8b0f66dd7b53cbeeb18bcae8b28e975 Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Thu, 21 Feb 2013 21:51:43 +0100 Subject: [PATCH 33/50] Dont download subtitles when output is stdout --- svtplay_dl.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/svtplay_dl.py b/svtplay_dl.py index 57b848f..93db2e7 100755 --- a/svtplay_dl.py +++ b/svtplay_dl.py @@ -807,7 +807,8 @@ class Urplay(): else: download_rtmp(options, rtmp) if options.subtitle: - subtitle_tt(options, subtitle) + if options.output != "-" + subtitle_tt(options, subtitle) class Qbrick(): def handle(self, url): @@ -903,7 +904,8 @@ class Kanal5(): options.other = "-W %s -y %s " % ("http://www.kanal5play.se/flash/StandardPlayer.swf", filename) download_rtmp(options, steambaseurl) if options.subtitle: - subtitle_json(options, subtitle) + if options.output != "-" + subtitle_json(options, subtitle) class Expressen(): def handle(self, url): @@ -1002,7 +1004,8 @@ class Viaplay(): options.other = "-W http://flvplayer.viastream.viasat.tv/play/swf/player110516.swf?rnd=1315434062" download_rtmp(options, filename) if options.subtitle and subtitle: - subtitle_sami(options, subtitle) + if options.output != "-" + subtitle_sami(options, subtitle) class Tv4play(): def handle(self, url): @@ -1071,7 +1074,8 @@ class Tv4play(): manifest = "%s?hdcore=2.8.0&g=hejsan" % test["path"] download_hds(options, manifest, swf) if options.subtitle and subtitle: - subtitle_smi(options, subtitle) + if options.output != "-" + subtitle_smi(options, subtitle) class Svtplay(): def handle(self, url): @@ -1141,7 +1145,8 @@ class Svtplay(): except KeyError: sys.exit(1) if len(suburl) > 0: - subtitle_wsrt(options, suburl) + if options.output != "-" + subtitle_wsrt(options, suburl) class Nrk(object): def handle(self, url): From 5f8c13d4e8aeaa113ca06aecb95b1ebfc9970083 Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Fri, 22 Feb 2013 10:05:37 +0100 Subject: [PATCH 34/50] Forgot to put an : on if-condition... --- svtplay_dl.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/svtplay_dl.py b/svtplay_dl.py index 93db2e7..65d4237 100755 --- a/svtplay_dl.py +++ b/svtplay_dl.py @@ -25,7 +25,7 @@ import struct import binascii from datetime import timedelta -__version__ = "0.9.2013.02.21" +__version__ = "0.9.2013.02.22" class Options: """ @@ -807,7 +807,7 @@ class Urplay(): else: download_rtmp(options, rtmp) if options.subtitle: - if options.output != "-" + if options.output != "-": subtitle_tt(options, subtitle) class Qbrick(): @@ -904,7 +904,7 @@ class Kanal5(): options.other = "-W %s -y %s " % ("http://www.kanal5play.se/flash/StandardPlayer.swf", filename) download_rtmp(options, steambaseurl) if options.subtitle: - if options.output != "-" + if options.output != "-": subtitle_json(options, subtitle) class Expressen(): @@ -1004,7 +1004,7 @@ class Viaplay(): options.other = "-W http://flvplayer.viastream.viasat.tv/play/swf/player110516.swf?rnd=1315434062" download_rtmp(options, filename) if options.subtitle and subtitle: - if options.output != "-" + if options.output != "-": subtitle_sami(options, subtitle) class Tv4play(): @@ -1074,7 +1074,7 @@ class Tv4play(): manifest = "%s?hdcore=2.8.0&g=hejsan" % test["path"] download_hds(options, manifest, swf) if options.subtitle and subtitle: - if options.output != "-" + if options.output != "-": subtitle_smi(options, subtitle) class Svtplay(): @@ -1145,7 +1145,7 @@ class Svtplay(): except KeyError: sys.exit(1) if len(suburl) > 0: - if options.output != "-" + if options.output != "-": subtitle_wsrt(options, suburl) class Nrk(object): From dd3096dc1304c3c1f4e41eaaf57661f1f392adfd Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Fri, 22 Feb 2013 10:11:09 +0100 Subject: [PATCH 35/50] use the right variable for the subtitles --- svtplay_dl.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/svtplay_dl.py b/svtplay_dl.py index 65d4237..ddb6ab5 100755 --- a/svtplay_dl.py +++ b/svtplay_dl.py @@ -578,7 +578,7 @@ def subtitle_tt(options, url): if filename: options.output = "%s.srt" % filename.group(1) log.info("Subtitle: %s", options.output) - fd = open(filename, "w") + fd = open(options.output, "w") fd.write(data) fd.close() @@ -595,7 +595,7 @@ def subtitle_json(options, url): if filename: options.output = "%s.srt" % filename.group(1) log.info("Subtitle: %s", options.output) - fd = open(filename, "w") + fd = open(options.output, "w") fd.write(subs) fd.close() @@ -617,7 +617,7 @@ def subtitle_sami(options, url): if filename: options.output = "%s.srt" % filename.group(1) log.info("Subtitle: %s", options.output) - fd = open(filename, "w") + fd = open(options.output, "w") fd.write(subs) fd.close() @@ -636,7 +636,7 @@ def subtitle_smi(options, url): if filename: options.output = "%s.srt" % filename.group(1) log.info("Subtitle: %s", options.output) - fd = open(filename, "w") + fd = open(options.output, "w") fd.write(subs) fd.close() @@ -655,7 +655,7 @@ def subtitle_wsrt(options, url): if filename: options.output = "%s.srt" % filename.group(1) log.info("Subtitle: %s", options.output) - fd = open(filename, "w") + fd = open(options.output, "w") fd.write(srt) fd.close() From 1ec5a6cf63dbdd864694b26d10261fc237e1e58c Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Tue, 26 Feb 2013 18:41:51 +0100 Subject: [PATCH 36/50] justin.tv: write an error message when we cant find any streams --- svtplay_dl.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/svtplay_dl.py b/svtplay_dl.py index ddb6ab5..704b90c 100755 --- a/svtplay_dl.py +++ b/svtplay_dl.py @@ -719,11 +719,15 @@ class Justin(): streams[int(i.find("video_height").text)] = stream except AttributeError: pass + if len(streams) > 0: + test = select_quality(options, streams) + options.other = "-j '%s' -W %s" % (test["token"], options.other) + options.resume = False + download_rtmp(options, test["url"]) + else: + log.error("Can't any streams") + sys.exit(2) - test = select_quality(options, streams) - options.other = "-j '%s' -W %s" % (test["token"], options.other) - options.resume = False - download_rtmp(options, test["url"]) class Hbo(): def handle(self, url): From c86bd9f96517b73854b695207f15f896d62fde1b Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Tue, 26 Feb 2013 22:01:35 +0100 Subject: [PATCH 37/50] kanal5: download_* should set the extention --- svtplay_dl.py | 1 - 1 file changed, 1 deletion(-) diff --git a/svtplay_dl.py b/svtplay_dl.py index 704b90c..68305b8 100755 --- a/svtplay_dl.py +++ b/svtplay_dl.py @@ -904,7 +904,6 @@ class Kanal5(): filename = test["source"] match = re.search("^(.*):", filename) - options.output = "%s.%s" % (options.output, match.group(1)) options.other = "-W %s -y %s " % ("http://www.kanal5play.se/flash/StandardPlayer.swf", filename) download_rtmp(options, steambaseurl) if options.subtitle: From c3bbc87eb7fb29ea5111113e449a77b75497da18 Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Tue, 26 Feb 2013 22:26:05 +0100 Subject: [PATCH 38/50] kanal5: HLS support --- svtplay_dl.py | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/svtplay_dl.py b/svtplay_dl.py index 68305b8..4eeea68 100755 --- a/svtplay_dl.py +++ b/svtplay_dl.py @@ -887,25 +887,33 @@ class Kanal5(): if not match: log.error("Can't find video file") sys.exit(2) - url = "http://www.kanal5play.se/api/getVideo?format=FLASH&videoId=%s" % match.group(1) + format = "FLASH" + if options.hls: + format = "IPHONE" + url = "http://www.kanal5play.se/api/getVideo?format=%s&videoId=%s" % (format, match.group(1)) data = json.loads(get_http_data(url)) options.live = data["isLive"] - steambaseurl = data["streamBaseUrl"] if data["hasSubtitle"]: subtitle = "http://www.kanal5play.se/api/subtitles/%s" % match.group(1) - streams = {} + if options.hls: + url = data["streams"][0]["source"] + baseurl = url[0:url.rfind("/")] + download_hls(options, url, baseurl) + else: + steambaseurl = data["streamBaseUrl"] + streams = {} - for i in data["streams"]: - stream = {} - stream["source"] = i["source"] - streams[int(i["bitrate"])] = stream + for i in data["streams"]: + stream = {} + stream["source"] = i["source"] + streams[int(i["bitrate"])] = stream - test = select_quality(options, streams) + test = select_quality(options, streams) - filename = test["source"] - match = re.search("^(.*):", filename) - options.other = "-W %s -y %s " % ("http://www.kanal5play.se/flash/StandardPlayer.swf", filename) - download_rtmp(options, steambaseurl) + filename = test["source"] + match = re.search("^(.*):", filename) + options.other = "-W %s -y %s " % ("http://www.kanal5play.se/flash/StandardPlayer.swf", filename) + download_rtmp(options, steambaseurl) if options.subtitle: if options.output != "-": subtitle_json(options, subtitle) From 60b3b2e290b165848d8333efd784aecea1b33412 Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Tue, 26 Feb 2013 22:31:51 +0100 Subject: [PATCH 39/50] download_hls: we are checking for http and not ttp. --- svtplay_dl.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/svtplay_dl.py b/svtplay_dl.py index 4eeea68..71c804b 100755 --- a/svtplay_dl.py +++ b/svtplay_dl.py @@ -25,7 +25,7 @@ import struct import binascii from datetime import timedelta -__version__ = "0.9.2013.02.22" +__version__ = "0.9.2013.02.26" class Options: """ @@ -404,7 +404,7 @@ def download_hls(options, url, baseurl=None): streams[int(i[1]["BANDWIDTH"])] = i[0] test = select_quality(options, streams) - if baseurl and test[1:4] != "http": + if baseurl and test[0:4] != "http": test = "%s%s" % (baseurl, test) m3u8 = get_http_data(test) globaldata, files = parsem3u(m3u8) From fc5874356c03e178e8001dc14395cb045bc142b5 Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Tue, 26 Feb 2013 23:25:08 +0100 Subject: [PATCH 40/50] kanal5: add a notice about drm hls files. --- svtplay_dl.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/svtplay_dl.py b/svtplay_dl.py index 71c804b..e36f73e 100755 --- a/svtplay_dl.py +++ b/svtplay_dl.py @@ -898,6 +898,9 @@ class Kanal5(): if options.hls: url = data["streams"][0]["source"] baseurl = url[0:url.rfind("/")] + if data["streams"][0]["drmProtected"]: + log.error("We cant download drm files for this site.") + sys.exit(2) download_hls(options, url, baseurl) else: steambaseurl = data["streamBaseUrl"] From 6f3889fadab4c8b017439cdab2ed837d5a5e30b5 Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Tue, 26 Feb 2013 23:28:59 +0100 Subject: [PATCH 41/50] kanal5: update .swf file --- svtplay_dl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/svtplay_dl.py b/svtplay_dl.py index e36f73e..a259e46 100755 --- a/svtplay_dl.py +++ b/svtplay_dl.py @@ -915,7 +915,7 @@ class Kanal5(): filename = test["source"] match = re.search("^(.*):", filename) - options.other = "-W %s -y %s " % ("http://www.kanal5play.se/flash/StandardPlayer.swf", filename) + options.other = "-W %s -y %s " % ("http://www.kanal5play.se/flash/K5StandardPlayer.swf", filename) download_rtmp(options, steambaseurl) if options.subtitle: if options.output != "-": From fafca242c1fbaf5464d901f8da3b432bceb141b9 Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Wed, 27 Feb 2013 18:06:24 +0100 Subject: [PATCH 42/50] generic: return the site when we found it --- svtplay_dl.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/svtplay_dl.py b/svtplay_dl.py index a259e46..44f9939 100755 --- a/svtplay_dl.py +++ b/svtplay_dl.py @@ -1263,9 +1263,7 @@ class generic(object): url = match.group(1) for i in sites: if i.handle(url): - stream = i - break - return url, stream + return url, i def progressbar(total, pos, msg=""): """ From f519935241ffa8803951542ac5ce046ae3043860 Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Tue, 5 Mar 2013 23:31:20 +0100 Subject: [PATCH 43/50] Support for vimeo files --- svtplay_dl.py | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/svtplay_dl.py b/svtplay_dl.py index 44f9939..1a9f057 100755 --- a/svtplay_dl.py +++ b/svtplay_dl.py @@ -1253,6 +1253,37 @@ class Radioplay(object): log.error("Can't find any streams.") sys.exit(2) +class Vimeo(object): + def handle(self, url): + return "vimeo.com" in url + + def get(self, options, url): + data = get_http_data(url, referer="") + match = data.split(' = {config:')[1].split(',assets:')[0] + if match: + jsondata = json.loads(match) + sig = jsondata['request']['signature'] + vidid = jsondata["video"]["id"] + timestamp = jsondata['request']['timestamp'] + referer = jsondata["request"]["referrer"] + avail_quality = jsondata["video"]["files"]["h264"] + selected_quality = None + for i in avail_quality: + if options.quality == i: + selected_quality = i + + if options.quality and selected_quality is None: + log.error("Can't find that quality. (Try one of: %s)", + ", ".join(map(str, avail_quality))) + sys.exit(4) + elif options.quality is None and selected_quality is None: + selected_quality = avail_quality[0] + url = "http://player.vimeo.com/play_redirect?clip_id=%s&sig=%s&time=%s&quality=%s&codecs=H264,VP8,VP6&type=moogaloop_local&embed_location=%s" % (vidid, sig, timestamp, selected_quality, referer) + download_http(options, url) + else: + log.error("Can't find any streams.") + sys.exit(2) + class generic(object): ''' Videos embed in sites ''' def get(self, sites, url): @@ -1301,7 +1332,7 @@ def progressbar(total, pos, msg=""): def get_media(url, options): sites = [Aftonbladet(), Dr(), Expressen(), Hbo(), Justin(), Kanal5(), Nrk(), - Qbrick(), Ruv(), Radioplay(), Sr(), Svtplay(), Tv4play(), Urplay(), Viaplay()] + Qbrick(), Ruv(), Radioplay(), Sr(), Svtplay(), Tv4play(), Urplay(), Viaplay(), Vimeo()] stream = None for i in sites: if i.handle(url): From 1a58b136ee482f7fb2daba00281b480c6ceec7b6 Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Tue, 5 Mar 2013 23:32:20 +0100 Subject: [PATCH 44/50] generic: support for embeded vimeo files. --- svtplay_dl.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/svtplay_dl.py b/svtplay_dl.py index 1a9f057..a2aa94b 100755 --- a/svtplay_dl.py +++ b/svtplay_dl.py @@ -1296,6 +1296,13 @@ class generic(object): if i.handle(url): return url, i + match = re.search("src=\"(http://player.vimeo.com/video/[0-9]+)\" ", data) + if match: + for i in sites: + if i.handle(match.group(1)): + return match.group(1), i + return url, stream + def progressbar(total, pos, msg=""): """ Given a total and a progress position, output a progress bar From f1ff63fa4f4dac1939690808e9b0df06d11d606e Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Tue, 5 Mar 2013 23:34:03 +0100 Subject: [PATCH 45/50] get_http_data: support for referer-header. --- svtplay_dl.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/svtplay_dl.py b/svtplay_dl.py index a2aa94b..db62cbf 100755 --- a/svtplay_dl.py +++ b/svtplay_dl.py @@ -268,7 +268,7 @@ def decode_f4f(fragID, fragData): start += tagLen + 11 + 4 return start -def get_http_data(url, method="GET", header="", data=""): +def get_http_data(url, method="GET", header="", data="", referer=None): """ Get the page to parse it for streams """ request = Request(url) request.add_header('User-Agent', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.9.0.3) Gecko/2008092417 Firefox/3.0.3') @@ -277,6 +277,8 @@ def get_http_data(url, method="GET", header="", data=""): request.add_header('Content-Type', header) if len(data) > 0: request.add_data(data) + if referer: + request.add_header('Referer', referer) try: response = urlopen(request) except HTTPError as e: From 4cca843143bab51c7e3bfc5d748b2760a48f182d Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Tue, 5 Mar 2013 23:35:00 +0100 Subject: [PATCH 46/50] download_http: adding a better U-A. need this for vimeo. --- svtplay_dl.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/svtplay_dl.py b/svtplay_dl.py index db62cbf..ac980cf 100755 --- a/svtplay_dl.py +++ b/svtplay_dl.py @@ -474,7 +474,14 @@ def download_hls(options, url, baseurl=None): def download_http(options, url): """ Get the stream from HTTP """ - response = urlopen(url) + request = Request(url) + request.add_header('User-Agent', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.9.0.3) Gecko/2008092417 Firefox/3.0.3') + try: + response = urlopen(request) + except HTTPError as e: + print "ERROR" + print e.reason + sys.exit() try: total_size = response.info()['Content-Length'] except KeyError: From a0443b856b34eabaa0a4cb40278b849e83dafc66 Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Wed, 6 Mar 2013 10:31:48 +0100 Subject: [PATCH 47/50] download_http: better error message. --- svtplay_dl.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/svtplay_dl.py b/svtplay_dl.py index ac980cf..b1abbb5 100755 --- a/svtplay_dl.py +++ b/svtplay_dl.py @@ -479,9 +479,9 @@ def download_http(options, url): try: response = urlopen(request) except HTTPError as e: - print "ERROR" - print e.reason - sys.exit() + log.error("Something wrong with that url") + log.error("Error code: %s" % e.code) + sys.exit(5) try: total_size = response.info()['Content-Length'] except KeyError: From 2f37fce3b5595d451bffa341e831fe872c6c6e78 Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Wed, 6 Mar 2013 10:35:00 +0100 Subject: [PATCH 48/50] download_http: add an extension if we cant find one --- svtplay_dl.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/svtplay_dl.py b/svtplay_dl.py index b1abbb5..425ef96 100755 --- a/svtplay_dl.py +++ b/svtplay_dl.py @@ -25,7 +25,7 @@ import struct import binascii from datetime import timedelta -__version__ = "0.9.2013.02.26" +__version__ = "0.9.2013.03.06" class Options: """ @@ -492,6 +492,8 @@ def download_http(options, url): extension = re.search("(\.[a-z0-9]+)$", url) if extension: options.output = options.output + extension.group(1) + else: + options.output = "%s.mp4" % options.output log.info("Outfile: %s", options.output) file_d = open(options.output, "wb") else: From 0f3e33e31dfd9c4c5e4b94758df3ad9986e74155 Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Wed, 6 Mar 2013 20:12:56 +0100 Subject: [PATCH 49/50] subtitle: download the data in the class --- svtplay_dl.py | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/svtplay_dl.py b/svtplay_dl.py index 425ef96..df7d74a 100755 --- a/svtplay_dl.py +++ b/svtplay_dl.py @@ -566,12 +566,11 @@ def norm(name): else: return name -def subtitle_tt(options, url): +def subtitle_tt(options, data): i = 1 data = "" skip = False - fh = get_http_data(url) - tree = ET.parse(fh) + tree = ET.parse(data) for node in tree.iter(): tag = norm(node.tag) if tag == "p": @@ -593,8 +592,8 @@ def subtitle_tt(options, url): fd.write(data) fd.close() -def subtitle_json(options, url): - data = json.loads(get_http_data(url)) +def subtitle_json(options, data): + data = json.loads(data) number = 1 subs = "" for i in data: @@ -610,8 +609,7 @@ def subtitle_json(options, url): fd.write(subs) fd.close() -def subtitle_sami(options, url): - data = get_http_data(url) +def subtitle_sami(options, data): tree = ET.XML(data) subt = tree.find("Font") subs = "" @@ -632,8 +630,7 @@ def subtitle_sami(options, url): fd.write(subs) fd.close() -def subtitle_smi(options, url): - data = get_http_data(url) +def subtitle_smi(options, data): recomp = re.compile(r'\s+

(.*)
\s+\s+

', re.M|re.I|re.U) number = 1 subs = "" @@ -651,8 +648,7 @@ def subtitle_smi(options, url): fd.write(subs) fd.close() -def subtitle_wsrt(options, url): - data = get_http_data(url) +def subtitle_wsrt(options, data): recomp = re.compile("(\d+)\r\n([\d:\.]+ --> [\d:\.]+)?([^\r\n]+)?\r\n([^\r\n]+)\r\n(([^\r\n]*)\r\n)?") srt = "" for i in recomp.finditer(data): @@ -823,7 +819,8 @@ class Urplay(): download_rtmp(options, rtmp) if options.subtitle: if options.output != "-": - subtitle_tt(options, subtitle) + data = get_http_data(subtitle) + subtitle_tt(options, data) class Qbrick(): def handle(self, url): @@ -930,7 +927,8 @@ class Kanal5(): download_rtmp(options, steambaseurl) if options.subtitle: if options.output != "-": - subtitle_json(options, subtitle) + data = get_http_data(data) + subtitle_json(options, data) class Expressen(): def handle(self, url): @@ -1030,7 +1028,8 @@ class Viaplay(): download_rtmp(options, filename) if options.subtitle and subtitle: if options.output != "-": - subtitle_sami(options, subtitle) + data = get_http_data(subtitle) + subtitle_sami(options, data) class Tv4play(): def handle(self, url): @@ -1100,7 +1099,8 @@ class Tv4play(): download_hds(options, manifest, swf) if options.subtitle and subtitle: if options.output != "-": - subtitle_smi(options, subtitle) + data = get_http_data(subtitle) + subtitle_smi(options, data) class Svtplay(): def handle(self, url): @@ -1171,7 +1171,8 @@ class Svtplay(): sys.exit(1) if len(suburl) > 0: if options.output != "-": - subtitle_wsrt(options, suburl) + data = get_http_data(suburl) + subtitle_wsrt(options, data) class Nrk(object): def handle(self, url): From f67caf39115b0e06e8533599c74c987565429917 Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Fri, 8 Mar 2013 17:53:41 +0100 Subject: [PATCH 50/50] We dont need pyamf anymore. --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 5fd1339..a042d5f 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,6 @@ You need: * [RTMPDump](http://rtmpdump.mplayerhq.hu/) 2.4 or higher * [PyCrypto](https://www.dlitz.net/software/pycrypto/) to download encrypted HLS streams -* [PyAMF](http://www.pyamf.org/) for kanal5.se / kanal9play.se This script works for: