diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index ab4fee266..000000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,72 +0,0 @@ -name: GitHub CI - -on: - pull_request: - push: - workflow_dispatch: - schedule: - - cron: 0 0 * * 0 - -defaults: - run: - shell: 'bash -Eeuo pipefail -x {0}' - -concurrency: - group: ${{ github.ref }} - cancel-in-progress: true - -jobs: - - generate-jobs: - name: Generate Jobs - runs-on: ubuntu-latest - outputs: - strategy: ${{ steps.generate-jobs.outputs.strategy }} - steps: - - uses: actions/checkout@v3 - - uses: docker-library/bashbrew@HEAD - - id: generate-jobs - name: Generate Jobs - run: | - strategy="$("$BASHBREW_SCRIPTS/github-actions/generate.sh")" - - # https://github.com/docker-library/python/pull/706 (ensure we don't have any unexpected ".a" leftovers in "/usr/local") - strategy="$(jq <<<"$strategy" -c ' - .matrix.include |= map( - if .os == "ubuntu-latest" then - .runs.test += "\n" + ( - .meta.entries - | map( - .tags[0] - | "aFiles=\"$(docker run --rm \(. | @sh) find /usr/local -name \"*.a\" | tee /dev/stderr)\"; [ -z \"$aFiles\" ]" - ) - | join("\n") - ) - else . end - ) - ')" - - EOF="EOF-$RANDOM-$RANDOM-$RANDOM" - echo "strategy<<$EOF" >> "$GITHUB_OUTPUT" - jq <<<"$strategy" . | tee -a "$GITHUB_OUTPUT" - echo "$EOF" >> "$GITHUB_OUTPUT" - - test: - needs: generate-jobs - strategy: ${{ fromJson(needs.generate-jobs.outputs.strategy) }} - name: ${{ matrix.name }} - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v3 - - name: Prepare Environment - run: ${{ matrix.runs.prepare }} - - name: Pull Dependencies - run: ${{ matrix.runs.pull }} - - name: Build ${{ matrix.name }} - run: ${{ matrix.runs.build }} - - name: History ${{ matrix.name }} - run: ${{ matrix.runs.history }} - - name: Test ${{ matrix.name }} - run: ${{ matrix.runs.test }} - - name: '"docker images"' - run: ${{ matrix.runs.images }} diff --git a/.github/workflows/verify-templating.yml b/.github/workflows/verify-templating.yml deleted file mode 100644 index 14497bec6..000000000 --- a/.github/workflows/verify-templating.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: Verify Templating - -on: - pull_request: - push: - -defaults: - run: - shell: 'bash -Eeuo pipefail -x {0}' - -jobs: - apply-templates: - name: Check For Uncomitted Changes - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Apply Templates - run: ./apply-templates.sh - - name: Check Git Status - run: | - status="$(git status --short)" - [ -z "$status" ] diff --git a/.scripts/create_build_name.sh b/.scripts/create_build_name.sh new file mode 100644 index 000000000..741a15bcc --- /dev/null +++ b/.scripts/create_build_name.sh @@ -0,0 +1 @@ +git diff --name-only | grep Dockerfile diff --git a/3.11/bullseye/Dockerfile b/3.11/bullseye/Dockerfile index f62b76b85..85ec83ad5 100644 --- a/3.11/bullseye/Dockerfile +++ b/3.11/bullseye/Dockerfile @@ -1,18 +1,18 @@ -# -# NOTE: THIS DOCKERFILE IS GENERATED VIA "apply-templates.sh" -# -# PLEASE DO NOT EDIT IT DIRECTLY. -# - -FROM buildpack-deps:bullseye +# 기존에 Ubuntu 를 base 로 하는 이미지에서 base image 만 nvidia 이미지로 스왑 +FROM --platform=linux/x86_64 nvidia/cuda:12.3.1-devel-ubuntu22.04 # ensure local python is preferred over distribution python ENV PATH /usr/local/bin:$PATH - # http://bugs.python.org/issue19846 # > At the moment, setting "LANG=C" on a Linux system *fundamentally breaks Python 3*, and that's not OK. ENV LANG C.UTF-8 +# Docker Image 상에서는 timezone 을 설정되지 않아 apt-get 단계에서 넘어가지 않는 케이스가 존재 +ENV TZ Asia/Seoul + +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime \ + && echo $TZ > /etc/timezone + # runtime dependencies RUN set -eux; \ apt-get update; \ @@ -20,6 +20,10 @@ RUN set -eux; \ libbluetooth-dev \ tk-dev \ uuid-dev \ + wget \ + libssl-dev openssl \ + libgl1-mesa-glx libffi-dev \ + libbz2-dev \ ; \ rm -rf /var/lib/apt/lists/* diff --git a/README.md b/README.md index 617f64ae7..be26c2372 100644 --- a/README.md +++ b/README.md @@ -1,26 +1,13 @@ -# https://github.com/docker-library/python +> Forked from https://github.com/docker-library/python -## Maintained by: [the Docker Community](https://github.com/docker-library/python) +# 목적 +* GPU 서버 기반의 python image 를 만들기 위해서 사용. 기존의 python Docker build 의 base image 를 GPU 기반으로 수정. +* 일부 테스트를 통해서 수정이 필요한 부분에 대해서도 대응됨. -This is the Git repo of the [Docker "Official Image"](https://github.com/docker-library/official-images#what-are-official-images) for [`python`](https://hub.docker.com/_/python/) (not to be confused with any official `python` image provided by `python` upstream). See [the Docker Hub page](https://hub.docker.com/_/python/) for the full readme on how to use this Docker image and for information regarding contributing and issues. +# how to build -The [full image description on Docker Hub](https://hub.docker.com/_/python/) is generated/maintained over in [the docker-library/docs repository](https://github.com/docker-library/docs), specifically in [the `python` directory](https://github.com/docker-library/docs/tree/master/python). +* [invz-customized-python-build-and-push.py](invz-customized-python-build-and-push.py) 를 실행 -## See a change merged here that doesn't show up on Docker Hub yet? -For more information about the full official images change lifecycle, see [the "An image's source changed in Git, now what?" FAQ entry](https://github.com/docker-library/faq#an-images-source-changed-in-git-now-what). - -For outstanding `python` image PRs, check [PRs with the "library/python" label on the official-images repository](https://github.com/docker-library/official-images/labels/library%2Fpython). For the current "source of truth" for [`python`](https://hub.docker.com/_/python/), see [the `library/python` file in the official-images repository](https://github.com/docker-library/official-images/blob/master/library/python). - ---- - -- [![build status badge](https://img.shields.io/github/actions/workflow/status/docker-library/python/ci.yml?branch=master&label=GitHub%20CI)](https://github.com/docker-library/python/actions?query=workflow%3A%22GitHub+CI%22+branch%3Amaster) -- [![build status badge](https://img.shields.io/jenkins/s/https/doi-janky.infosiftr.net/job/update.sh/job/python.svg?label=Automated%20update.sh)](https://doi-janky.infosiftr.net/job/update.sh/job/python/) - -| Build | Status | Badges | (per-arch) | -|:-:|:-:|:-:|:-:| -| [![amd64 build status badge](https://img.shields.io/jenkins/s/https/doi-janky.infosiftr.net/job/multiarch/job/amd64/job/python.svg?label=amd64)](https://doi-janky.infosiftr.net/job/multiarch/job/amd64/job/python/) | [![arm32v5 build status badge](https://img.shields.io/jenkins/s/https/doi-janky.infosiftr.net/job/multiarch/job/arm32v5/job/python.svg?label=arm32v5)](https://doi-janky.infosiftr.net/job/multiarch/job/arm32v5/job/python/) | [![arm32v6 build status badge](https://img.shields.io/jenkins/s/https/doi-janky.infosiftr.net/job/multiarch/job/arm32v6/job/python.svg?label=arm32v6)](https://doi-janky.infosiftr.net/job/multiarch/job/arm32v6/job/python/) | [![arm32v7 build status badge](https://img.shields.io/jenkins/s/https/doi-janky.infosiftr.net/job/multiarch/job/arm32v7/job/python.svg?label=arm32v7)](https://doi-janky.infosiftr.net/job/multiarch/job/arm32v7/job/python/) | -| [![arm64v8 build status badge](https://img.shields.io/jenkins/s/https/doi-janky.infosiftr.net/job/multiarch/job/arm64v8/job/python.svg?label=arm64v8)](https://doi-janky.infosiftr.net/job/multiarch/job/arm64v8/job/python/) | [![i386 build status badge](https://img.shields.io/jenkins/s/https/doi-janky.infosiftr.net/job/multiarch/job/i386/job/python.svg?label=i386)](https://doi-janky.infosiftr.net/job/multiarch/job/i386/job/python/) | [![mips64le build status badge](https://img.shields.io/jenkins/s/https/doi-janky.infosiftr.net/job/multiarch/job/mips64le/job/python.svg?label=mips64le)](https://doi-janky.infosiftr.net/job/multiarch/job/mips64le/job/python/) | [![ppc64le build status badge](https://img.shields.io/jenkins/s/https/doi-janky.infosiftr.net/job/multiarch/job/ppc64le/job/python.svg?label=ppc64le)](https://doi-janky.infosiftr.net/job/multiarch/job/ppc64le/job/python/) | -| [![s390x build status badge](https://img.shields.io/jenkins/s/https/doi-janky.infosiftr.net/job/multiarch/job/s390x/job/python.svg?label=s390x)](https://doi-janky.infosiftr.net/job/multiarch/job/s390x/job/python/) | [![windows-amd64 build status badge](https://img.shields.io/jenkins/s/https/doi-janky.infosiftr.net/job/multiarch/job/windows-amd64/job/python.svg?label=windows-amd64)](https://doi-janky.infosiftr.net/job/multiarch/job/windows-amd64/job/python/) | [![put-shared build status badge](https://img.shields.io/jenkins/s/https/doi-janky.infosiftr.net/job/put-shared/job/light/job/python.svg?label=put-shared)](https://doi-janky.infosiftr.net/job/put-shared/job/light/job/python/) | - - +### 원본 레포지토리 +> https://github.com/docker-library/python diff --git a/invz-customized-python-build-and-push.py b/invz-customized-python-build-and-push.py new file mode 100644 index 000000000..f51d85bca --- /dev/null +++ b/invz-customized-python-build-and-push.py @@ -0,0 +1,99 @@ +""" +innerverz 서비스를 위해서 수정한 dockerfile 들을 배포. +배포 후에는 반드시 커밋 및 푸시 필요. +또한 배포의 타깃은 +""" +import asyncio +import dataclasses +import os +from textwrap import dedent + +TRUE_LIKE_STR_SET = {"true", "1", "t", "y", "yes", "yeah", "yup", "certainly", "uh-huh"} + + +@dataclasses.dataclass +class CONTROLS: + DRYRUN = os.environ.get("DRYRUN") in TRUE_LIKE_STR_SET + ASYNC = os.environ.get("ASYNC") in TRUE_LIKE_STR_SET + BUILD_ALL = os.environ.get("BUILD_ALL") in TRUE_LIKE_STR_SET + + +INVZ_CUSTOMIZED_PYTHON_DOCKERFILE = [ + "3.11/bullseye/Dockerfile", +] +CUSTOMIZED_SUFFIX = "--invz-cust" +REGION = "us-east-1" +ECR_REGISTRY = f"346614530986.dkr.ecr.{REGION}.amazonaws.com" + + +async def run_command_with_env(cmd, env_vars): + # Create a new environment dictionary + new_env = os.environ.copy() + new_env.update(env_vars) + + # Run the command + process = await asyncio.create_subprocess_shell( + cmd, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + env=new_env + ) + + # Wait for the command to complete + stdout, stderr = await process.communicate() + + return stdout.decode(), stderr.decode() + + +def dockerfile_to_tag(dockerfile_str: str) -> str: + return "-".join(dockerfile_str.split("/")[:-1]) + CUSTOMIZED_SUFFIX + + +def form_docker_build_and_push_cmd(dockerfile, tag): + image = f"{ECR_REGISTRY}/python:{tag}" + + cmd_form = dedent(f"""\ + docker pull {image} + docker build -f {dockerfile} -t {image} . + docker push {image} + """) + return cmd_form + + +async def _gather_job(cmds): + return await asyncio.gather(*[run_command_with_env(cmd, {}) for cmd in cmds]) + + +def get_target_dockerfiles(build_all: bool): + if build_all: + return INVZ_CUSTOMIZED_PYTHON_DOCKERFILE + all_diffs, stderr = asyncio.run(run_command_with_env("git diff --name-only", {})) + print(f"all git diffs. `{all_diffs}`") + return [ + d for d in all_diffs.split("\n") + if d in INVZ_CUSTOMIZED_PYTHON_DOCKERFILE + ] + + +if __name__ == '__main__': + target_dockerfiles = get_target_dockerfiles(os.environ.get("build_all", False)) + docker_login_cmd = (f"aws ecr get-login-password --region {REGION}" + f" | docker login --username AWS --password-stdin {ECR_REGISTRY}") + print("asyncio.run(run_command_with_env(docker_login_cmd, {}))") + print("docker_login_cmd", docker_login_cmd) + if not os.environ.get("dryrun"): + print(asyncio.run(run_command_with_env(docker_login_cmd, {}))) + + docker_build_and_push_cmds = [ + form_docker_build_and_push_cmd(dockerfile=dockerfile, tag=dockerfile_to_tag(dockerfile)) + for dockerfile in target_dockerfiles + ] + print(docker_build_and_push_cmds) + print("asyncio.gather(*[run_command_with_env(cmd, {}) for cmd in docker_build_and_push_cmds])") + if os.environ.get("dryrun"): + pass + elif os.environ.get("async"): + asyncio.run(_gather_job(docker_build_and_push_cmds)) + else: + for cmd in docker_build_and_push_cmds: + os.system(cmd)