From 9256e4232f6619d4c146ccb877465b899c83f45a Mon Sep 17 00:00:00 2001 From: Roxedus Date: Wed, 24 Feb 2021 14:58:26 +0100 Subject: [PATCH] Init commit --- .gitignore | 3 + .vscode/extensions.json | 8 ++ .vscode/settings.json | 11 +++ README.md | 20 +++++ alpine.Dockerfile | 36 ++++++++ build.sh | 15 ++++ generator/example.env | 7 ++ generator/main.py | 153 ++++++++++++++++++++++++++++++++ generator/requirements.txt | 2 + generator/templates/template.j2 | 13 +++ requirements.txt | 2 + setup.cfg | 21 +++++ 12 files changed, 291 insertions(+) create mode 100644 .gitignore create mode 100644 .vscode/extensions.json create mode 100644 .vscode/settings.json create mode 100644 README.md create mode 100644 alpine.Dockerfile create mode 100644 build.sh create mode 100644 generator/example.env create mode 100644 generator/main.py create mode 100644 generator/requirements.txt create mode 100644 generator/templates/template.j2 create mode 100644 requirements.txt create mode 100644 setup.cfg diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..baef6aa --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +.dev/ +venv/ +out/ \ No newline at end of file diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 0000000..e50f452 --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,8 @@ +{ + "recommendations": [ + "ms-python.python", + "ms-python.vscode-pylance", + "oderwat.indent-rainbow", + "timonwong.shellcheck" + ] +} \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..3667bac --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,11 @@ +{ + "python.languageSever": "Pylance", + "python.linting.enabled": true, + "python.linting.pylintEnabled": true, + "python.linting.flake8Enabled": true, + "python.linting.pycodestyleEnabled": true, + "[python]": { + "editor.rulers": [120], + }, + "editor.tabSize": 4 +} \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..d96bcda --- /dev/null +++ b/README.md @@ -0,0 +1,20 @@ +# Wheelie + +Current flow: + + 1. Use `alpine.Dockerfile` to create a build-enviroment. + 2. Generate wheels with docker run. + + ```bash + docker run -v "$PWD/out:/build" wheelie \ + --find-links=https://rox-wheels.s3.eu-north-1.amazonaws.com/alpine/3.13 \ + --trusted-host rox-wheels.s3-website.eu-north-1.amazonaws.com \ + cryptography==3.4.1 + ``` + + 3. Upload the wheels to S3 with `generator/main.py`. Takes S3 info as enviroment variables. + + ```bash + cd generator + python3 main.py -d alpine -r 3.13 -l -a + ``` diff --git a/alpine.Dockerfile b/alpine.Dockerfile new file mode 100644 index 0000000..3ca049e --- /dev/null +++ b/alpine.Dockerfile @@ -0,0 +1,36 @@ +ARG REL + +FROM ghcr.io/linuxserver/baseimage-alpine:${REL} + +RUN \ + echo "**** install build packages ****" && \ + apk add --no-cache --virtual=build-dependencies \ + autoconf \ + build-base \ + ca-certificates \ + cargo \ + cmake \ + curl \ + cython \ + g++ \ + gcc \ + git \ + glib-dev \ + gnupg \ + jq \ + libffi-dev \ + libstdc++ \ + libxml2-dev \ + libxslt \ + libxslt-dev \ + linux-headers \ + scons \ + make \ + openssl \ + openssl-dev \ + py3-pip \ + python3-dev && \ + pip3 install -U --no-cache-dir pip wheel setuptools && \ + mkdir -p /build + +ENTRYPOINT [ "pip3", "wheel", "--wheel-dir=/build", "--find-links=/build", "--no-cache-dir" ] diff --git a/build.sh b/build.sh new file mode 100644 index 0000000..ff90259 --- /dev/null +++ b/build.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +BUCKET=https://rox-wheels.s3.eu-north-1.amazonaws.com/alpine/3.13 + +REL=3.12 + +docker build -t wheelie -f alpine.Dockerfile --build-arg=REL=${REL} . + +while IFS="" read -r p || [ -n "$p" ] +do + docker run -v "$PWD/out:/build" wheelie --find-links=$BUCKET --trusted-host rox-wheels.s3-website.eu-north-1.amazonaws.com "$p" +done < requirements.txt + +cd generator || return +python3 main.py -d alpine -r ${REL} -l -a \ No newline at end of file diff --git a/generator/example.env b/generator/example.env new file mode 100644 index 0000000..fb41d28 --- /dev/null +++ b/generator/example.env @@ -0,0 +1,7 @@ +S3_REGION= +S3_BUCKET= + +S3_ACCESS_KEY= +S3_SECRET_KEY= + +S3_IS_MINIO= \ No newline at end of file diff --git a/generator/main.py b/generator/main.py new file mode 100644 index 0000000..af18e07 --- /dev/null +++ b/generator/main.py @@ -0,0 +1,153 @@ +from argparse import ArgumentParser +from io import BytesIO +from json import dumps, loads +from ntpath import basename +from os import environ, listdir +from os.path import isfile, join + +import botocore.exceptions as botoexeptions +from boto3 import client +from jinja2 import Environment, FileSystemLoader + +if not environ.get("CI"): + from dotenv import load_dotenv + load_dotenv(dotenv_path="../.dev/.env") + +mime = {"json": "application/json", + "whl": "application/x-wheel+zip", # Not a "real" type + "html": "text/html" + } + + +class Bucket(): + def __init__(self) -> None: + self.client = client( + "s3", + region_name=environ.get("S3_REGION"), + aws_access_key_id=environ.get("S3_ACCESS_KEY"), + aws_secret_access_key=environ.get("S3_SECRET_KEY"), + endpoint_url=environ.get("S3_IS_MINIO", None) + ) + self.bucket_name = environ.get("S3_BUCKET") + self._list() + + def _list(self) -> list: + response = self.client.list_buckets() + buckets = [] + for bucket in response["Buckets"]: + buckets.append(bucket) + return buckets + + def download(self, bucket, filepath) -> bytes: + _file = BytesIO() + self.client.download_fileobj(bucket, filepath, _file) + _file.seek(0) + return _file.read() + + def upload(self, bucket, filepath, file_): + m_type = mime[filepath.split(".")[-1]] + self.client.upload_fileobj(file_, bucket, filepath, ExtraArgs={ + "ContentType": m_type, "ACL": "public-read", "CacheControl": "no-cache"}) + + +class Index(Bucket): + def __init__(self, distribution: str, release: str, folder: str) -> None: + super().__init__() + self.distro = distribution + self.rel = release + self.path = folder + self.get_manifest() + self._gather() + self.loader = FileSystemLoader("templates") + self.jinja = Environment(loader=self.loader) + if environ.get("S3_IS_MINIO"): + self.upload_listing(self.create_listing( + ["alpine/", "ubuntu/"] + ), path_="index.html") + self.upload_listing(self.create_listing( + ["3.13/", "3.12/"] + ), path_="alpine/index.html") + self.upload_listing(self.create_listing( + ["bionic/", "focal/"] + ), path_="ubuntu/index.html") + + def _gather(self) -> list: + existing_files = [] + for key in self.manifest: + existing_files.append(key) + new_files = [] + for file_ in listdir(self.path): + _dir = join(self.path, file_) + if isfile(_dir) and not file_.endswith("any.whl"): + file_name = basename(_dir) + if file_name not in existing_files: + new_files.append(file_name) + self.new_files = new_files + self.files = existing_files + new_files + self.files.sort(key=lambda v: v.upper()) + + def get_manifest(self) -> dict: + filepath = f"{self.distro}/{self.rel}/manifest.json" + try: + self.manifest = loads(self.download(self.bucket_name, filepath).decode("UTF-8")) + except botoexeptions.ClientError: + self.upload_manifest({}) + self.manifest = {} + + def upload_manifest(self, obj: dict): + filepath = f"{self.distro}/{self.rel}/manifest.json" + obj = bytes(dumps(obj), "ascii") + file_ = BytesIO(obj) + file_.seek(0) + self.upload(self.bucket_name, filepath, file_) + + def add(self): + filepath = f"{self.distro}/{self.rel}/" + for wheel in self.new_files: + file_ = open(self.path + wheel, "rb") + self.upload(self.bucket_name, filepath + wheel, file_) + self.manifest[wheel] = {"distribution": self.distro, "release": self.rel} + self.upload_manifest(self.manifest) + self.upload_listing(self.create_listing(self.files)) + + def create_listing(self, wheels: list) -> str: + template = self.jinja.get_template("template.j2") + output = template.render(wheels=wheels) + return output + + def upload_listing(self, obj: str, path_=None): + filepath = path_ if path_ else f"{self.distro}/{self.rel}/index.html" + obj = bytes(obj, "ascii") + file_ = BytesIO(obj) + file_.seek(0) + self.upload(self.bucket_name, filepath, file_) + + +if __name__ == "__main__": + parser = ArgumentParser(prog="Wheelie", description="Program to upload and present wheels to S3 buckets") + + parser.add_argument("-d", "--distribution", type=str, + help="Distribution the wheels are target for.", choices=["alpine", "ubuntu"], required=True) + + parser.add_argument("-r", "--release", type=str, + help="Release of the distribution.", choices=["3.12", "3.13", "focal"], required=True) + + parser.add_argument("-p", "--path", type=str, default="../out/", + help="Path to the wheels") + + parser.add_argument("-l", "--list", action="store_true", help="Lists files that would be added") + parser.add_argument("-a", "--add", action="store_true", help="Adds new files, and builds the index") + + args = parser.parse_args() + + distro = args.distribution + rel = args.release + folder = args.path + + inx = Index(distribution=distro, release=rel, folder=folder) + + if args.list: + print(inx.new_files if inx.new_files != [] else "No new files") + + if args.add: + inx.add() diff --git a/generator/requirements.txt b/generator/requirements.txt new file mode 100644 index 0000000..ae18985 --- /dev/null +++ b/generator/requirements.txt @@ -0,0 +1,2 @@ +jinja2 +boto3 \ No newline at end of file diff --git a/generator/templates/template.j2 b/generator/templates/template.j2 new file mode 100644 index 0000000..def9f9d --- /dev/null +++ b/generator/templates/template.j2 @@ -0,0 +1,13 @@ + + + wheels.linuxserver.io + + +

Precompiled wheels commonly used in Linuxserver's images

+ + {%- for wheel in wheels %} + {{ wheel | e }} +
+ {%- endfor %} + + \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..9e7d472 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +cryptography==3.4.1 +PyNaCl==1.4.0 \ No newline at end of file diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..5501f2c --- /dev/null +++ b/setup.cfg @@ -0,0 +1,21 @@ +[flake8] +max-line-length = 120 +count = True +statistics = True + +[isort] +no_lines_before = LOCALFOLDER +indent = " " +line_length = 120 +balanced_wrapping = True +multi_line_output = 4 + +[pycodestyle] +max_line_length = 120 + +[pylint.FORMAT] +max-line-length=120 + +[pylint.BASIC] +variable-naming-style=camelCase +constant-naming-style=camelCase \ No newline at end of file