From b1b15f57e9f216271329775c6845d0724ed84e41 Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Fri, 4 Jan 2019 23:43:37 +0100 Subject: [PATCH] Cibuild: first version. Upload releases and snapshots to aws --- .travis.yml | 46 +++++++++------- appveyor.yml | 30 +++++------ dockerfile/Dockerfile | 6 ++- scripts/cibuild.py | 120 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 166 insertions(+), 36 deletions(-) create mode 100755 scripts/cibuild.py diff --git a/.travis.yml b/.travis.yml index 890fb06..21e41e2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,28 +1,34 @@ sudo: false language: python -python: - - 3.4 - - 3.5 - - 3.6 - # see https://github.com/travis-ci/travis-ci/issues/9815 - # 3.7 +matrix: + fast_finish: true + include: + - python: 3.5 + - python: 3.6 + env: CIBUILD=1 + - python: 3.7 + dist: xenial + sudo: true # required workaround for https://github.com/travis-ci/travis-ci/issues/9815 cache: pip - install: - - pip install -r requirements-dev.txt -r requirements.txt - +- pip install -r requirements-dev.txt -r requirements.txt awscli script: - - flake8 --jobs 8 lib/svtplay_dl - - nosetests -v --all-modules --with-doctest svtplay_dl - -deploy: - provider: pypi - user: spaam - twine_version: 1.12.1 - password: - secure: YMxYmNOexO13ORBYPFPQrbt9RiLgRPmhTxHRuBVkrICXc1AXli+YGX56hF9nabCFsBT+OEEiGEux5QwSPXGDs2SoQWPKkcCS7uxD3l9QmZuA39IfwsaoDfPziU3Mr+heAmM8WdTls6rFEbUJEMG1/bNVqmijOWDBUi/k9MAC2N4= - on: - tags: true +- flake8 --jobs 8 lib/svtplay_dl +- nosetests -v --all-modules --with-doctest svtplay_dl +- make +- | + if [[ $CIBUILD == "1" ]] + then + python scripts/cibuild.py + fi +env: + matrix: + - secure: NqhAYBLkalelOFcSNdTaBuqXtQ2r0m9Lt5/V2tDZdtqPjur0H12DLsDIjt/X0jID6ci6LmdbXNZ97wz3D3KjBBeAuw5VRKiiZMPZTuOB5GdFRP/H6PmZYazGPHU2idFJLwG9JUhy+LMyzhpwyw/hruV1a2Udd7VT5fiPkf2hZ7A= + - secure: bmNNj7iM/SFKROFVh6s9q877a2z+VtieCkmtdV8SzYfR1KwG27TwxRckSx9JKWfKMri+PvcF9jz9SFt7IMP3SJxM22pYVKKPJa164ze0IuQLAlhsrdNISe2CPOMJrXAq2SP6RTn/YfZnyv3vjxuLJDuG4IPD1Yosln+gBZgCvjk= + - secure: AVwpii9rbPfEZ2HpanR9DMRf+0ica2BMui454XL4sgdXkfzkocOvw46ztT+XNpb25fpcfTC/hDOtyFeobNFH8E2DreM8h8kn7pqFCJeGAS8OkbDESAS2HApe1VqJj8xMQCKdZz/B25Ye1Lannqamxrt5Nai6lgQaaBeaXifb6/Q= + - secure: ZUK56NpCC5CVUVTY6mWwFlcJxK8/WFf+0g/x71cp6eO7j8+yW78UahcT2ismphVgzvGgYRZCksC2Y/qyl1HzwuKE73gPIcvcD9MFOnGmrTedln78+CLRjZh6+YG4VhIgWhRikI3tGMZ7iChj04Qz6yBBG1hOY4T2oO/Uiv96NOY= + - secure: keZjlq/Op5xD6ku2rAfvxr7waSCOkHgOJw0d14vBYmWyFtivdzUHeIliQlgsdmzKpGn1qOUXP9l52pYmCkSIwV1xZ6+gyxB5KH52rzs505whtvQOLsMXJeSxhpkI1WcX+4Rc3VUPnirbNyfkImghs07wSGAAMIGs8rbSsbHj/2U= + - secure: b4m5le6Q494cPzXHMmBKjyF7UN+cxLs1cn7CnzpHh7dHsAqH0rP+GtuhOj23eTSpFrkUFnA7ofyDF/3SRi1Qij14KeQEj9Ri/fTGGPoOYgvVnds9mbqqWeqzMbVEguxp0gz8F6g8TfUTgt9eFAYkWKnQS5Ar3o56Zh2jT7gBHNo= diff --git a/appveyor.yml b/appveyor.yml index fe2d612..577a9c9 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -10,7 +10,7 @@ install: - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%" - "python -m pip install -U pip setuptools" - "pip install -r requirements-dev.txt -r requirements.txt" - - "pip install cx_freeze" + - "pip install cx_freeze awscli" - "git describe --tags --dirty --always" test_script: @@ -23,23 +23,23 @@ after_test: - "python %PYTHON%\\Scripts\\cxfreeze --include-modules=queue,idna.idnadata --target-dir=svtplay-dl bin/svtplay-dl" - "svtplay-dl\\svtplay-dl.exe --version" - "7z.exe a -tzip svtplay-dl.zip svtplay-dl" + - "python scripts/cibuild.py" artifacts: - path: svtplay-dl.zip name: svtplay-dl - -deploy: - - provider: FTP - protocol: sftp - host: srv.svtplay-dl.se - username: web - password: - secure: wdvTOWRd6TXhtgfSPmYea8XOT/5sc4s3xhELbowqT58= - folder: /var/www/download/$(APPVEYOR_REPO_TAG_NAME) - - artifact: svtplay-dl.zip - on: - appveyor_repo_tag: true - +environment: + TWINE_USERNAME: + secure: yig0ualP0M39xIahEXhxZA== + TWINE_PASSWORD: + secure: 79wxWyPcqzj6131wKAuWBg== + DOCKER_USERNAME: + secure: yig0ualP0M39xIahEXhxZA== + DOCKER_PASSWORD: + secure: uiu9FbvYYz89WrBtF6BN0w== + AWS_ACCESS_KEY_ID: + secure: ED96t0LSsl1VnMu5CEQddJYdsIwIdsomF6gmY7r49w8= + AWS_SECRET_ACCESS_KEY: + secure: r/GpKgKWeTtq2k2J+Wy4qwOvLa8auzCgSYvEb88NqzenKZh04wUiWyxwXk6xc7UV build: off \ No newline at end of file diff --git a/dockerfile/Dockerfile b/dockerfile/Dockerfile index e184892..416aaf5 100644 --- a/dockerfile/Dockerfile +++ b/dockerfile/Dockerfile @@ -2,6 +2,9 @@ # using testing repo to get pycryptodome FROM alpine:edge MAINTAINER spaam + +COPY dist/*.whl . + RUN set -xe \ && echo 'http://dl-cdn.alpinelinux.org/alpine/edge/testing' >> /etc/apk/repositories \ && apk add --no-cache \ @@ -11,7 +14,8 @@ RUN set -xe \ rtmpdump \ py3-pycryptodome \ ffmpeg \ - && pip3 install svtplay-dl + && python3 -m pip install *.whl \ + && rm -f *.whl WORKDIR /data diff --git a/scripts/cibuild.py b/scripts/cibuild.py new file mode 100755 index 0000000..4ff0233 --- /dev/null +++ b/scripts/cibuild.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 + +import subprocess +import argparse +import os +import logging +import sys +import glob +from datetime import datetime + +root = os.path.normpath(os.path.join(os.path.dirname(__file__), "..")) +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger("cibuild") + + +parser = argparse.ArgumentParser(prog="cibuild") +general = parser.add_argument_group() +general.add_argument("-s", "--snapshots", action="store_true", dest="snapshot", default=False) +general.add_argument("-r", "--release", action="store_true", dest="release", default=False) +options = parser.parse_args() + + +twine_username = os.environ.get("TWINE_USERNAME") +twine_password = os.environ.get("TWINE_PASSWORD") +docker_username = os.environ.get("DOCKER_USERNAME") +docker_password = os.environ.get("DOCKER_PASSWORD") +aws_creds = os.environ.get("AWS_ACCESS_KEY_ID") + +travis_tag = os.environ.get("TRAVIS_TAG", "") +travis_branch = os.environ.get("TRAVIS_BRANCH", "") +appveyor_tag = os.environ.get("APPVEYOR_REPO_TAG_NAME", "") +appveyor_branch = os.environ.get("APPVEYOR_REPO_BRANCH", "") + + +def tag(): + return travis_tag or appveyor_tag + + +def branch(): + return travis_branch or appveyor_branch + + +def docker_name(): + if tag(): + ver = tag() + else: + ver = "dev" + return "spaam/svtplay-dl:{}".format(ver) + + +def build_docker(): + logger.info("Building docker") + subprocess.check_output([ + "docker", "build", "-f", "dockerfile/Dockerfile", "-t", docker_name(), "." + ]) + subprocess.check_call([ + "docker", "login", "-u", docker_username, "-p", docker_password + ]) + subprocess.check_call([ + "docker", "push", docker_name() + ]) + + +def build_package(): + logger.info("Building python package") + subprocess.check_output([ + "python3", "setup.py", "-q", "sdist", "bdist_wheel" + ]) + + +def snapshot_folder(): + """ + Use the commit date in UTC as folder name + """ + logger.info("Snapshot folder") + try: + stdout = subprocess.check_output(["git", "show", "-s", "--format=%cI", "HEAD"]) + except subprocess.CalledProcessError as e: + logger.error("Error: {}".format(e.output.decode('ascii', 'ignore').strip())) + sys.exit(2) + except FileNotFoundError as e: + logger.error("Error: {}".format(e)) + sys.exit(2) + ds = stdout.decode('ascii', 'ignore').strip() + dt = datetime.fromisoformat(ds) + utc = dt - dt.utcoffset() + return utc.strftime("%Y%m%d_%H%M%S") + + +def aws_upload(): + if tag(): + folder = "release" + version = tag() + else: + folder = "snapshots" + version = snapshot_folder() + logger.info("Upload to aws {}/{}".format(folder, version)) + for file in ["svtplay-dl", "svtplay-dl.zip"]: + if os.path.isfile(file): + subprocess.check_call([ + "aws", "s3", "cp", "{}".format(file), "s3://svtplay-dl/{}/{}/{}".format(folder, version, file) + ]) + + +def pypi_upload(): + logger.info("Uploading to pypi") + sdist = glob.glob(os.path.join("dist/", 'svtplay_dl-*.tar.gz'))[0] + subprocess.check_call(["twine", "upload", sdist]) + + +if branch() is not "master": + sys.exit(0) + + +build_package() +build_docker() +aws_upload() + +if tag(): + pypi_upload()