From 55bf8edcbeebe92bdf079e53836f82398c993d31 Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Sun, 9 May 2021 01:16:16 +0200 Subject: [PATCH] hds is dead. thanks for the time. No one is using this anymore. flash is dead --- lib/svtplay_dl/fetcher/hds.py | 331 --------------------- lib/svtplay_dl/service/bigbrother.py | 11 - lib/svtplay_dl/service/nrk.py | 12 - lib/svtplay_dl/service/oppetarkiv.py | 18 -- lib/svtplay_dl/service/raw.py | 5 - lib/svtplay_dl/service/vg.py | 10 - lib/svtplay_dl/tests/test_protocol_prio.py | 6 +- lib/svtplay_dl/utils/getmedia.py | 2 +- lib/svtplay_dl/utils/parser.py | 4 +- lib/svtplay_dl/utils/stream.py | 6 +- 10 files changed, 9 insertions(+), 396 deletions(-) delete mode 100644 lib/svtplay_dl/fetcher/hds.py diff --git a/lib/svtplay_dl/fetcher/hds.py b/lib/svtplay_dl/fetcher/hds.py deleted file mode 100644 index 9f35aa2..0000000 --- a/lib/svtplay_dl/fetcher/hds.py +++ /dev/null @@ -1,331 +0,0 @@ -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- -import base64 -import binascii -import copy -import struct -import xml.etree.ElementTree as ET -from urllib.parse import urlparse - -from svtplay_dl.error import ServiceError -from svtplay_dl.error import UIException -from svtplay_dl.fetcher import VideoRetriever -from svtplay_dl.utils.output import ETA -from svtplay_dl.utils.output import output -from svtplay_dl.utils.output import progress_stream -from svtplay_dl.utils.output import progressbar - - -def _chr(temp): - return chr(temp) - - -class HDSException(UIException): - def __init__(self, url, message): - self.url = url - super().__init__(message) - - -class LiveHDSException(HDSException): - def __init__(self, url): - super().__init__(url, "This is a live HDS stream, and they are not supported.") - - -def hdsparse(config, res, manifest, output=None): - streams = {} - bootstrap = {} - - if not res: - return streams - - if res.status_code >= 400: - streams[0] = ServiceError(f"Can't read HDS playlist. {res.status_code}") - return streams - data = res.text - - xml = ET.XML(data) - - bootstrapIter = xml.iter("{http://ns.adobe.com/f4m/1.0}bootstrapInfo") - mediaIter = xml.iter("{http://ns.adobe.com/f4m/1.0}media") - - if xml.find("{http://ns.adobe.com/f4m/1.0}drmAdditionalHeader") is not None: - streams[0] = ServiceError("HDS DRM protected content.") - return streams - for i in bootstrapIter: - if "id" in i.attrib: - bootstrap[i.attrib["id"]] = i.text - else: - bootstrap["0"] = i.text - parse = urlparse(manifest) - querystring = parse.query - url = f"{parse.scheme}://{parse.netloc}{parse.path}" - for i in mediaIter: - bootstrapid = bootstrap[i.attrib["bootstrapInfoId"]] - streams[int(i.attrib["bitrate"])] = HDS( - copy.copy(config), - url, - i.attrib["bitrate"], - url_id=i.attrib["url"], - bootstrap=bootstrapid, - metadata=i.find("{http://ns.adobe.com/f4m/1.0}metadata").text, - querystring=querystring, - cookies=res.cookies, - output=output, - ) - return streams - - -class HDS(VideoRetriever): - @property - def name(self): - return "hds" - - def download(self): - self.output_extention = "flv" - if self.config.get("live") and not self.config.get("force"): - raise LiveHDSException(self.url) - - querystring = self.kwargs["querystring"] - cookies = self.kwargs["cookies"] - bootstrap = base64.b64decode(self.kwargs["bootstrap"]) - box = readboxtype(bootstrap, 0) - antal = None - if box[2] == b"abst": - antal = readbox(bootstrap, box[0]) - baseurl = self.url[0 : self.url.rfind("/")] - - file_d = output(self.output, self.config, "flv") - if file_d is None: - return - - metasize = struct.pack(">L", len(base64.b64decode(self.kwargs["metadata"])))[1:] - file_d.write(binascii.a2b_hex(b"464c560105000000090000000012")) - file_d.write(metasize) - file_d.write(binascii.a2b_hex(b"00000000000000")) - file_d.write(base64.b64decode(self.kwargs["metadata"])) - file_d.write(binascii.a2b_hex(b"00000000")) - i = 1 - start = antal[1]["first"] - total = antal[1]["total"] - eta = ETA(total) - while i <= total: - url = "{}/{}Seg1-Frag{}?{}".format(baseurl, self.kwargs["url_id"], start, querystring) - if not self.config.get("silent"): - eta.update(i) - progressbar(total, i, "".join(["ETA: ", str(eta)])) - data = self.http.request("get", url, cookies=cookies) - if data.status_code == 404: - break - data = data.content - number = decode_f4f(i, data) - file_d.write(data[number:]) - i += 1 - start += 1 - - file_d.close() - if not self.config.get("silent"): - progress_stream.write("\n") - self.finished = True - - -def readbyte(data, pos): - return struct.unpack("B", bytes(_chr(data[pos]), "ascii"))[0] - - -def read16(data, pos): - endpos = pos + 2 - return struct.unpack(">H", data[pos:endpos])[0] - - -def read24(data, pos): - end = pos + 3 - return struct.unpack(">L", "\x00" + data[pos:end])[0] - - -def read32(data, pos): - end = pos + 4 - return struct.unpack(">i", data[pos:end])[0] - - -def readu32(data, pos): - end = pos + 4 - return struct.unpack(">I", data[pos:end])[0] - - -def read64(data, pos): - end = pos + 8 - return struct.unpack(">Q", data[pos:end])[0] - - -def readstring(data, pos): - length = 0 - while bytes(_chr(data[pos + length]), "ascii") != b"\x00": - length += 1 - endpos = pos + length - string = data[pos:endpos] - pos += length + 1 - return pos, string - - -def readboxtype(data, pos): - boxsize = read32(data, pos) - tpos = pos + 4 - endpos = tpos + 4 - boxtype = data[tpos:endpos] - if boxsize > 1: - boxsize -= 8 - pos += 8 - return pos, boxsize, boxtype - - -# Note! A lot of variable assignments are commented out. These are -# accessible values that we currently don't use. -def readbox(data, pos): - # version = readbyte(data, pos) - pos += 1 - # flags = read24(data, pos) - pos += 3 - # bootstrapversion = read32(data, pos) - pos += 4 - # byte = readbyte(data, pos) - pos += 1 - # profile = (byte & 0xC0) >> 6 - # live = (byte & 0x20) >> 5 - # update = (byte & 0x10) >> 4 - # timescale = read32(data, pos) - pos += 4 - # currentmediatime = read64(data, pos) - pos += 8 - # smptetimecodeoffset = read64(data, pos) - pos += 8 - temp = readstring(data, pos) - # movieidentifier = temp[1] - pos = temp[0] - serverentrycount = readbyte(data, pos) - pos += 1 - serverentrytable = [] - i = 0 - while i < serverentrycount: - temp = readstring(data, pos) - serverentrytable.append(temp[1]) - pos = temp[0] - i += 1 - qualityentrycount = readbyte(data, pos) - pos += 1 - qualityentrytable = [] - i = 0 - while i < qualityentrycount: - temp = readstring(data, pos) - qualityentrytable.append(temp[1]) - pos = temp[0] - i += 1 - - tmp = readstring(data, pos) - # drm = tmp[1] - pos = tmp[0] - tmp = readstring(data, pos) - # metadata = tmp[1] - pos = tmp[0] - segmentruntable = readbyte(data, pos) - pos += 1 - if segmentruntable > 0: - tmp = readboxtype(data, pos) - boxtype = tmp[2] - boxsize = tmp[1] - pos = tmp[0] - if boxtype == b"asrt": - antal = readasrtbox(data, pos) - pos += boxsize - fragRunTableCount = readbyte(data, pos) - pos += 1 - i = 0 - first = 1 - while i < fragRunTableCount: - tmp = readboxtype(data, pos) - boxtype = tmp[2] - boxsize = tmp[1] - pos = tmp[0] - if boxtype == b"afrt": - first = readafrtbox(data, pos) - pos += boxsize - i += 1 - antal[1]["first"] = first - return antal - - -# Note! A lot of variable assignments are commented out. These are -# accessible values that we currently don't use. -def readafrtbox(data, pos): - # version = readbyte(data, pos) - pos += 1 - # flags = read24(data, pos) - pos += 3 - # timescale = read32(data, pos) - pos += 4 - qualityentry = readbyte(data, pos) - pos += 1 - i = 0 - while i < qualityentry: - temp = readstring(data, pos) - # qualitysegmulti = temp[1] - pos = temp[0] - i += 1 - fragrunentrycount = read32(data, pos) - pos += 4 - i = 0 - first = 1 - skip = False - while i < fragrunentrycount: - firstfragment = readu32(data, pos) - if not skip: - first = firstfragment - skip = True - pos += 4 - # timestamp = read64(data, pos) - pos += 8 - # duration = read32(data, pos) - pos += 4 - i += 1 - return first - - -# Note! A lot of variable assignments are commented out. These are -# accessible values that we currently don't use. -def readasrtbox(data, pos): - # version = readbyte(data, pos) - pos += 1 - # flags = read24(data, pos) - pos += 3 - qualityentrycount = readbyte(data, pos) - pos += 1 - qualitysegmentmodifers = [] - i = 0 - while i < qualityentrycount: - temp = readstring(data, pos) - qualitysegmentmodifers.append(temp[1]) - pos = temp[0] - i += 1 - - seqCount = read32(data, pos) - pos += 4 - ret = {} - i = 0 - - while i < seqCount: - firstseg = read32(data, pos) - pos += 4 - fragPerSeg = read32(data, pos) - pos += 4 - tmp = i + 1 - ret[tmp] = {"first": firstseg, "total": fragPerSeg} - i += 1 - return ret - - -def decode_f4f(fragID, fragData): - start = fragData.find(b"mdat") + 4 - if fragID > 1: - (tagLen,) = struct.unpack_from(">L", fragData, start) - tagLen &= 0x00FFFFFF - start += tagLen + 11 + 4 - return start diff --git a/lib/svtplay_dl/service/bigbrother.py b/lib/svtplay_dl/service/bigbrother.py index 1b7fa2a..4962625 100644 --- a/lib/svtplay_dl/service/bigbrother.py +++ b/lib/svtplay_dl/service/bigbrother.py @@ -5,7 +5,6 @@ import json import re from svtplay_dl.error import ServiceError -from svtplay_dl.fetcher.hds import hdsparse from svtplay_dl.fetcher.hls import hlsparse from svtplay_dl.fetcher.http import HTTP from svtplay_dl.service import OpenGraphThumbMixin @@ -59,16 +58,6 @@ class Bigbrother(Service, OpenGraphThumbMixin): return for i in renditions: - if i["defaultURL"].endswith("f4m"): - streams = hdsparse( - copy.copy(self.config), - self.http.request("get", i["defaultURL"], params={"hdcore": "3.7.0"}), - i["defaultURL"], - output=self.output, - ) - for n in list(streams.keys()): - yield streams[n] - if i["defaultURL"].endswith("m3u8"): streams = hlsparse(self.config, self.http.request("get", i["defaultURL"]), i["defaultURL"], output=self.output) for n in list(streams.keys()): diff --git a/lib/svtplay_dl/service/nrk.py b/lib/svtplay_dl/service/nrk.py index c95ad9b..fbaba85 100644 --- a/lib/svtplay_dl/service/nrk.py +++ b/lib/svtplay_dl/service/nrk.py @@ -5,7 +5,6 @@ import json import re from svtplay_dl.error import ServiceError -from svtplay_dl.fetcher.hds import hdsparse from svtplay_dl.fetcher.hls import hlsparse from svtplay_dl.service import OpenGraphThumbMixin from svtplay_dl.service import Service @@ -50,14 +49,3 @@ class Nrk(Service, OpenGraphThumbMixin): if streams: for n in list(streams.keys()): yield streams[n] - - else: - streams = hdsparse( - copy.copy(self.config), - self.http.request("get", manifest_url, params={"hdcore": "3.7.0"}), - manifest_url, - output=self.output, - ) - if streams: - for n in list(streams.keys()): - yield streams[n] diff --git a/lib/svtplay_dl/service/oppetarkiv.py b/lib/svtplay_dl/service/oppetarkiv.py index 138ebba..dcd936d 100644 --- a/lib/svtplay_dl/service/oppetarkiv.py +++ b/lib/svtplay_dl/service/oppetarkiv.py @@ -9,7 +9,6 @@ from urllib.parse import urlparse from svtplay_dl.error import ServiceError from svtplay_dl.fetcher.dash import dashparse -from svtplay_dl.fetcher.hds import hdsparse from svtplay_dl.fetcher.hls import hlsparse from svtplay_dl.service import OpenGraphThumbMixin from svtplay_dl.service import Service @@ -62,23 +61,6 @@ class OppetArkiv(Service, OpenGraphThumbMixin): streams = hlsparse(self.config, self.http.request("get", alt.request.url), alt.request.url, output=self.output) for n in list(streams.keys()): yield streams[n] - if i["format"] == "hds" or i["format"] == "flash": - match = re.search(r"\/se\/secure\/", i["url"]) - if not match: - streams = hdsparse(self.config, self.http.request("get", i["url"], params={"hdcore": "3.7.0"}), i["url"], output=self.output) - for n in list(streams.keys()): - yield streams[n] - if "alt" in query and len(query["alt"]) > 0: - alt = self.http.get(query["alt"][0]) - if alt: - streams = hdsparse( - self.config, - self.http.request("get", alt.request.url, params={"hdcore": "3.7.0"}), - alt.request.url, - output=self.output, - ) - for n in list(streams.keys()): - yield streams[n] if i["format"] == "dash264" or i["format"] == "dashhbbtv": streams = dashparse(self.config, self.http.request("get", i["url"]), i["url"], output=self.output) for n in list(streams.keys()): diff --git a/lib/svtplay_dl/service/raw.py b/lib/svtplay_dl/service/raw.py index 33d0d2f..11c8e37 100644 --- a/lib/svtplay_dl/service/raw.py +++ b/lib/svtplay_dl/service/raw.py @@ -2,7 +2,6 @@ import os import re from svtplay_dl.fetcher.dash import dashparse -from svtplay_dl.fetcher.hds import hdsparse from svtplay_dl.fetcher.hls import hlsparse from svtplay_dl.service import Service @@ -13,10 +12,6 @@ class Raw(Service): self.output["title"] = filename streams = [] - if re.search(".f4m", self.url): - self.output["ext"] = "flv" - streams.append(hdsparse(self.config, self.http.request("get", self.url, params={"hdcore": "3.7.0"}), self.url, output=self.output)) - if re.search(".m3u8", self.url): streams.append(hlsparse(self.config, self.http.request("get", self.url), self.url, output=self.output)) diff --git a/lib/svtplay_dl/service/vg.py b/lib/svtplay_dl/service/vg.py index d833093..43926ee 100644 --- a/lib/svtplay_dl/service/vg.py +++ b/lib/svtplay_dl/service/vg.py @@ -4,7 +4,6 @@ import re from urllib.parse import urlparse from svtplay_dl.error import ServiceError -from svtplay_dl.fetcher.hds import hdsparse from svtplay_dl.fetcher.hls import hlsparse from svtplay_dl.fetcher.http import HTTP from svtplay_dl.service import OpenGraphThumbMixin @@ -28,15 +27,6 @@ class Vg(Service, OpenGraphThumbMixin): jsondata = json.loads(data) self.output["title"] = jsondata["title"] - if "hds" in jsondata["streamUrls"]: - streams = hdsparse( - self.config, - self.http.request("get", jsondata["streamUrls"]["hds"], params={"hdcore": "3.7.0"}), - jsondata["streamUrls"]["hds"], - output=self.output, - ) - for n in list(streams.keys()): - yield streams[n] if "hls" in jsondata["streamUrls"]: streams = hlsparse( self.config, diff --git a/lib/svtplay_dl/tests/test_protocol_prio.py b/lib/svtplay_dl/tests/test_protocol_prio.py index 4fdaa1e..a382b63 100644 --- a/lib/svtplay_dl/tests/test_protocol_prio.py +++ b/lib/svtplay_dl/tests/test_protocol_prio.py @@ -26,10 +26,10 @@ class PrioStreamsTest(unittest.TestCase): return [str(x) for x in protocol_prio(streams, ordered, **kwargs)] == expected def test_custom_order(self): - assert self._gen_proto_case(["http", "hds", "hls"], ["hds", "hls", "http"]) + assert self._gen_proto_case(["http", "hls"], ["hls", "http"]) def test_custom_order_1(self): - assert self._gen_proto_case(["http"], ["hds", "hls", "http"]) + assert self._gen_proto_case(["http"], ["hls", "http"]) def test_proto_unavail(self): - assert self._gen_proto_case(["http", "hds"], ["hls", "https"], expected=[]) + assert self._gen_proto_case(["http"], ["hls", "https"], expected=[]) diff --git a/lib/svtplay_dl/utils/getmedia.py b/lib/svtplay_dl/utils/getmedia.py index e531975..5784b59 100644 --- a/lib/svtplay_dl/utils/getmedia.py +++ b/lib/svtplay_dl/utils/getmedia.py @@ -229,7 +229,7 @@ def get_one_media(stream): if fstream.audio and post.detect: post.merge() if fstream.audio and not post.detect and fstream.finished: - logging.warning("Cant find ffmpeg/avconv. audio and video is in seperate files. if you dont want this use -P hls or hds") + logging.warning("Cant find ffmpeg/avconv. audio and video is in seperate files. if you dont want this use -P hls") if fstream.name == "hls" or fstream.config.get("remux"): post.remux() if fstream.config.get("silent_semi") and fstream.finished: diff --git a/lib/svtplay_dl/utils/parser.py b/lib/svtplay_dl/utils/parser.py index 1a4d276..e2df14e 100644 --- a/lib/svtplay_dl/utils/parser.py +++ b/lib/svtplay_dl/utils/parser.py @@ -176,13 +176,13 @@ def gen_parser(version="unknown"): dest="flexibleq", help="allow given quality (as above) to differ by an amount", ) - quality.add_argument("-P", "--preferred", default=None, metavar="preferred", help="preferred download method (dash, hls, hds, or http)") + quality.add_argument("-P", "--preferred", default=None, metavar="preferred", help="preferred download method (dash, hls, or http)") quality.add_argument("--list-quality", dest="list_quality", action="store_true", default=False, help="list the quality for a video") quality.add_argument( "--stream-priority", dest="stream_prio", default=None, - metavar="dash,hls,hds,http", + metavar="dash,hls,http", help="If two streams have the same quality, choose the one you prefer", ) quality.add_argument( diff --git a/lib/svtplay_dl/utils/stream.py b/lib/svtplay_dl/utils/stream.py index 66a6d59..f5d816c 100644 --- a/lib/svtplay_dl/utils/stream.py +++ b/lib/svtplay_dl/utils/stream.py @@ -6,8 +6,8 @@ from svtplay_dl.utils.http import HTTP # TODO: should be set as the default option in the argument parsing? -DEFAULT_PROTOCOL_PRIO = ["dash", "hls", "hds", "http"] -LIVE_PROTOCOL_PRIO = ["hls", "dash", "hds", "http"] +DEFAULT_PROTOCOL_PRIO = ["dash", "hls", "http"] +LIVE_PROTOCOL_PRIO = ["hls", "dash", "http"] DEFAULT_FORMAT_PRIO = ["h264", "h264-51"] @@ -80,7 +80,7 @@ def select_quality(config, streams): form_prio = DEFAULT_FORMAT_PRIO streams = format_prio(streams, form_prio) - # Extract protocol prio, in the form of "hls,hds,http", + # Extract protocol prio, in the form of "hls,http", # we want it as a list if config.get("stream_prio"):