Cibuild: first version.

Upload releases and snapshots to aws

.travis.yml

@@ -1,28 +1,34 @@
 sudo: false
 language: python
 
-python:
-  - 3.4
-  - 3.5
-  - 3.6
-  # see https://github.com/travis-ci/travis-ci/issues/9815
-  # 3.7
+matrix:
+  fast_finish: true
+  include:
+    - python: 3.5
+    - python: 3.6
+      env: CIBUILD=1
+    - python: 3.7
+      dist: xenial
+      sudo: true # required workaround for https://github.com/travis-ci/travis-ci/issues/9815
 
 cache: pip
-
 install:
-  - pip install -r requirements-dev.txt -r requirements.txt
-
+- pip install -r requirements-dev.txt -r requirements.txt awscli
 script:
-  - flake8 --jobs 8 lib/svtplay_dl
-  - nosetests -v --all-modules --with-doctest svtplay_dl
-
-deploy:
-  provider: pypi
-  user: spaam
-  twine_version: 1.12.1
-  password:
-    secure: YMxYmNOexO13ORBYPFPQrbt9RiLgRPmhTxHRuBVkrICXc1AXli+YGX56hF9nabCFsBT+OEEiGEux5QwSPXGDs2SoQWPKkcCS7uxD3l9QmZuA39IfwsaoDfPziU3Mr+heAmM8WdTls6rFEbUJEMG1/bNVqmijOWDBUi/k9MAC2N4=
-  on:
-    tags: true
+- flake8 --jobs 8 lib/svtplay_dl
+- nosetests -v --all-modules --with-doctest svtplay_dl
+- make
+- |
+  if [[ $CIBUILD == "1" ]]
+  then
+      python scripts/cibuild.py
+  fi
 
+env:
+  matrix:
+  - secure: NqhAYBLkalelOFcSNdTaBuqXtQ2r0m9Lt5/V2tDZdtqPjur0H12DLsDIjt/X0jID6ci6LmdbXNZ97wz3D3KjBBeAuw5VRKiiZMPZTuOB5GdFRP/H6PmZYazGPHU2idFJLwG9JUhy+LMyzhpwyw/hruV1a2Udd7VT5fiPkf2hZ7A=
+  - secure: bmNNj7iM/SFKROFVh6s9q877a2z+VtieCkmtdV8SzYfR1KwG27TwxRckSx9JKWfKMri+PvcF9jz9SFt7IMP3SJxM22pYVKKPJa164ze0IuQLAlhsrdNISe2CPOMJrXAq2SP6RTn/YfZnyv3vjxuLJDuG4IPD1Yosln+gBZgCvjk=
+  - secure: AVwpii9rbPfEZ2HpanR9DMRf+0ica2BMui454XL4sgdXkfzkocOvw46ztT+XNpb25fpcfTC/hDOtyFeobNFH8E2DreM8h8kn7pqFCJeGAS8OkbDESAS2HApe1VqJj8xMQCKdZz/B25Ye1Lannqamxrt5Nai6lgQaaBeaXifb6/Q=
+  - secure: ZUK56NpCC5CVUVTY6mWwFlcJxK8/WFf+0g/x71cp6eO7j8+yW78UahcT2ismphVgzvGgYRZCksC2Y/qyl1HzwuKE73gPIcvcD9MFOnGmrTedln78+CLRjZh6+YG4VhIgWhRikI3tGMZ7iChj04Qz6yBBG1hOY4T2oO/Uiv96NOY=
+  - secure: keZjlq/Op5xD6ku2rAfvxr7waSCOkHgOJw0d14vBYmWyFtivdzUHeIliQlgsdmzKpGn1qOUXP9l52pYmCkSIwV1xZ6+gyxB5KH52rzs505whtvQOLsMXJeSxhpkI1WcX+4Rc3VUPnirbNyfkImghs07wSGAAMIGs8rbSsbHj/2U=
+  - secure: b4m5le6Q494cPzXHMmBKjyF7UN+cxLs1cn7CnzpHh7dHsAqH0rP+GtuhOj23eTSpFrkUFnA7ofyDF/3SRi1Qij14KeQEj9Ri/fTGGPoOYgvVnds9mbqqWeqzMbVEguxp0gz8F6g8TfUTgt9eFAYkWKnQS5Ar3o56Zh2jT7gBHNo=

appveyor.yml

@@ -10,7 +10,7 @@ install:
   - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
   - "python -m pip install -U pip setuptools"
   - "pip install -r requirements-dev.txt -r requirements.txt"
-  - "pip install cx_freeze"
+  - "pip install cx_freeze awscli"
   - "git describe --tags --dirty --always"
 
 test_script:
@@ -23,23 +23,23 @@ after_test:
   - "python %PYTHON%\\Scripts\\cxfreeze --include-modules=queue,idna.idnadata --target-dir=svtplay-dl bin/svtplay-dl"
   - "svtplay-dl\\svtplay-dl.exe --version"
   - "7z.exe a -tzip svtplay-dl.zip svtplay-dl"
+  - "python scripts/cibuild.py"
 
 artifacts:
   - path: svtplay-dl.zip
     name: svtplay-dl
 
-
-deploy:
-  - provider: FTP
-    protocol: sftp
-    host: srv.svtplay-dl.se
-    username: web
-    password:
-      secure: wdvTOWRd6TXhtgfSPmYea8XOT/5sc4s3xhELbowqT58=
-    folder: /var/www/download/$(APPVEYOR_REPO_TAG_NAME)
-
-    artifact: svtplay-dl.zip
-    on:
-      appveyor_repo_tag: true
-
+environment:
+  TWINE_USERNAME:
+    secure: yig0ualP0M39xIahEXhxZA==
+  TWINE_PASSWORD:
+    secure: 79wxWyPcqzj6131wKAuWBg==
+  DOCKER_USERNAME:
+    secure: yig0ualP0M39xIahEXhxZA==
+  DOCKER_PASSWORD:
+    secure: uiu9FbvYYz89WrBtF6BN0w==
+  AWS_ACCESS_KEY_ID:
+    secure: ED96t0LSsl1VnMu5CEQddJYdsIwIdsomF6gmY7r49w8=
+  AWS_SECRET_ACCESS_KEY:
+    secure: r/GpKgKWeTtq2k2J+Wy4qwOvLa8auzCgSYvEb88NqzenKZh04wUiWyxwXk6xc7UV
 build: off

dockerfile/Dockerfile

@@ -2,6 +2,9 @@
 # using testing repo to get pycryptodome
 FROM alpine:edge
 MAINTAINER spaam
+
+COPY dist/*.whl .
+
 RUN set -xe \
 	&& echo 'http://dl-cdn.alpinelinux.org/alpine/edge/testing' >> /etc/apk/repositories  \
     && apk add --no-cache \
@@ -11,7 +14,8 @@ RUN set -xe \
         rtmpdump \
         py3-pycryptodome \
         ffmpeg \
-    && pip3 install svtplay-dl
+    && python3 -m pip install *.whl \
+    && rm -f *.whl
 
 WORKDIR /data
 

scripts/cibuild.py

@@ -0,0 +1,120 @@
+#!/usr/bin/env python3
+
+import subprocess
+import argparse
+import os
+import logging
+import sys
+import glob
+from datetime import datetime
+
+root = os.path.normpath(os.path.join(os.path.dirname(__file__), ".."))
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger("cibuild")
+
+
+parser = argparse.ArgumentParser(prog="cibuild")
+general = parser.add_argument_group()
+general.add_argument("-s", "--snapshots", action="store_true", dest="snapshot", default=False)
+general.add_argument("-r", "--release", action="store_true", dest="release", default=False)
+options = parser.parse_args()
+
+
+twine_username = os.environ.get("TWINE_USERNAME")
+twine_password = os.environ.get("TWINE_PASSWORD")
+docker_username = os.environ.get("DOCKER_USERNAME")
+docker_password = os.environ.get("DOCKER_PASSWORD")
+aws_creds = os.environ.get("AWS_ACCESS_KEY_ID")
+
+travis_tag = os.environ.get("TRAVIS_TAG", "")
+travis_branch = os.environ.get("TRAVIS_BRANCH", "")
+appveyor_tag = os.environ.get("APPVEYOR_REPO_TAG_NAME", "")
+appveyor_branch = os.environ.get("APPVEYOR_REPO_BRANCH", "")
+
+
+def tag():
+    return travis_tag or appveyor_tag
+
+
+def branch():
+    return travis_branch or appveyor_branch
+
+
+def docker_name():
+    if tag():
+        ver = tag()
+    else:
+        ver = "dev"
+    return "spaam/svtplay-dl:{}".format(ver)
+
+
+def build_docker():
+    logger.info("Building docker")
+    subprocess.check_output([
+        "docker", "build", "-f", "dockerfile/Dockerfile", "-t", docker_name(), "."
+    ])
+    subprocess.check_call([
+        "docker", "login", "-u", docker_username, "-p", docker_password
+    ])
+    subprocess.check_call([
+        "docker", "push", docker_name()
+    ])
+
+
+def build_package():
+    logger.info("Building python package")
+    subprocess.check_output([
+        "python3", "setup.py", "-q", "sdist", "bdist_wheel"
+    ])
+
+
+def snapshot_folder():
+    """
+    Use the commit date in UTC as folder name
+    """
+    logger.info("Snapshot folder")
+    try:
+        stdout = subprocess.check_output(["git", "show", "-s", "--format=%cI", "HEAD"])
+    except subprocess.CalledProcessError as e:
+        logger.error("Error: {}".format(e.output.decode('ascii', 'ignore').strip()))
+        sys.exit(2)
+    except FileNotFoundError as e:
+        logger.error("Error: {}".format(e))
+        sys.exit(2)
+    ds = stdout.decode('ascii', 'ignore').strip()
+    dt = datetime.fromisoformat(ds)
+    utc = dt - dt.utcoffset()
+    return utc.strftime("%Y%m%d_%H%M%S")
+
+
+def aws_upload():
+    if tag():
+        folder = "release"
+        version = tag()
+    else:
+        folder = "snapshots"
+        version = snapshot_folder()
+    logger.info("Upload to aws {}/{}".format(folder, version))
+    for file in ["svtplay-dl", "svtplay-dl.zip"]:
+        if os.path.isfile(file):
+            subprocess.check_call([
+                "aws", "s3", "cp", "{}".format(file), "s3://svtplay-dl/{}/{}/{}".format(folder, version, file)
+            ])
+
+
+def pypi_upload():
+    logger.info("Uploading to pypi")
+    sdist = glob.glob(os.path.join("dist/", 'svtplay_dl-*.tar.gz'))[0]
+    subprocess.check_call(["twine", "upload", sdist])
+
+
+if branch() is not "master":
+    sys.exit(0)
+
+
+build_package()
+build_docker()
+aws_upload()
+
+if tag():
+    pypi_upload()