diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000..d0ab1d0 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,47 @@ +version: 2.1 +executors: + docker-publisher: + environment: + IMAGE_NAME: tedivm/gitconsensus_service + docker: + - image: circleci/buildpack-deps:stretch +jobs: + build: + executor: docker-publisher + steps: + - checkout + - setup_remote_docker + - run: + name: Build Docker image + command: | + docker build -t $IMAGE_NAME:latest . | cat + + publish-latest: + executor: docker-publisher + steps: + - checkout + - setup_remote_docker + - run: + name: Build Docker image + command: | + docker build -t $IMAGE_NAME:latest . | cat + - run: + name: Publish Docker Image to Docker Hub + command: | + echo "$DOCKERHUB_PASS" | docker login -u "$DOCKERHUB_USERNAME" --password-stdin + docker push $IMAGE_NAME:latest | cat + +workflows: + version: 2 + build: + jobs: + - build: + filters: + branches: + ignore: master + publish: + jobs: + - publish-latest: + filters: + branches: + only: master \ No newline at end of file diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..8b25384 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,5 @@ +.git +venv +build +dist +*.pem diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..7eac12c --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2018 Robert Hafner + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Makefile b/Makefile index b45b0ad..57843b3 100644 --- a/Makefile +++ b/Makefile @@ -10,12 +10,25 @@ fresh: clean dependencies fulluninstall: uninstall clean -install: +testenv: clean_testenv + docker-compose up --build + +clean_testenv: + docker-compose down + +build_image: + docker build -f $(ROOT_DIR)/dockerfile -t tedivm/gitconsensus_service:latest . + +publish_image: build_image + docker push tedivm/gitconsensus_service:latest dependencies: if [ ! -d $(ROOT_DIR)/venv ]; then python3 -m venv $(ROOT_DIR)/venv; fi source $(ROOT_DIR)/venv/bin/activate; python -m pip install wheel; yes w | python -m pip install -e . +upgrade_dependencies: venv + source $(ROOT_DIR)/venv/bin/activate; ./bin/upgrade_dependencies.sh $(ROOT_DIR)/requirements.txt + application: if [ ! -d $(ROOT_DIR)/venv ]; then python3 -m venv $(ROOT_DIR)/venv; fi source $(ROOT_DIR)/venv/bin/activate; python -m pip install wheel; yes w | python -m pip install -r requirements.txt diff --git a/bin/upgrade_dependencies.sh b/bin/upgrade_dependencies.sh new file mode 100755 index 0000000..a7398fb --- /dev/null +++ b/bin/upgrade_dependencies.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +REQUIREMENTS_FILE=$1 + +while read p; do + if [[ -z "${p// }" ]];then + continue + fi + if [[ "${p// }" == \#* ]];then + continue + fi + + count=$(echo ${p// } | wc -m) + if (( count < 3 )); then + continue + fi + + program=$(echo $p | cut -d= -f1) + curr_version=$(cat ./requirements.txt | grep "${program}==" | cut -d'=' -f3 | sed 's/[^0-9.]*//g') + PACKAGE_JSON_URL="https://pypi.python.org/pypi/${program}/json" + version=$(curl -L -s "$PACKAGE_JSON_URL" | jq -r '.releases | keys | .[]' | sed '/^[^[:alpha:]]*$/!d' | sort -V | tail -1 | sed 's/[^0-9.]*//g' ) + + if [ "$curr_version" != "$version" ]; then + echo "Updating requirements for ${program} from ${curr_version} to ${version}." + sed -i -e "s/${program}==.*/${program}==${version}/g" $REQUIREMENTS_FILE + fi + +done < $REQUIREMENTS_FILE + +# OSX Creates a backup file. +rm -f "${REQUIREMENTS_FILE}-e" diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..0fdab38 --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,20 @@ +version: "3" + +services: + worker: + build: + context: . + dockerfile: dockerfile + volumes: + - ./gitconsensusservice:/app/gitconsensusservice + - ./github_app.private-key.pem:/app/github_app.private-key.pem + environment: + - 'CELERY_BROKER=pyamqp://guest@rabbitmq//' + - 'DEBUG=true' + - 'GITHUB_APP_ID=9231' + - 'PROCESS_INSTALLS_INTERVAL=30' + depends_on: + - rabbitmq + + rabbitmq: + image: rabbitmq diff --git a/docker/start_worker.sh b/docker/start_worker.sh new file mode 100755 index 0000000..03f1164 --- /dev/null +++ b/docker/start_worker.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +if [[ ! -v 'MINIMUM_WORKERS' ]]; then + MINIMUM_WORKERS=2 +fi + +if [[ ! -v 'MAXIMUM_WORKERS' ]]; then + MAXIMUM_WORKERS=10 +fi + +if [ "$DISABLE_BEAT" = "true" ] +then + echo 'Launching celery worker without beat' + celery -A gitconsensusservice.worker.celery worker --loglevel=info --autoscale=$MAXIMUM_WORKERS,$MINIMUM_WORKERS +else + echo 'Launching celery worker with beat enabled' + rm -f ~/celerybeat-schedule + celery -A gitconsensusservice.worker.celery worker --loglevel=info --autoscale=$MAXIMUM_WORKERS,$MINIMUM_WORKERS -B -s ~/celerybeat-schedule +fi diff --git a/dockerfile b/dockerfile new file mode 100644 index 0000000..b0b3dce --- /dev/null +++ b/dockerfile @@ -0,0 +1,17 @@ +FROM python:3.6 + +ADD requirements.txt /app/requirements.txt +WORKDIR /app/ +RUN pip install -r requirements.txt + +ENV SETTINGS /app/settings.yaml +ENV GITHUB_PRIVATE_KEY /app/github_app.private-key.pem + +ADD ./gitconsensusservice/ /app/gitconsensusservice + +RUN useradd -ms /bin/bash gitconsensusservice +USER gitconsensusservice + +ADD ./docker/start_worker.sh /home/gitconsensusservice/start_worker.sh + +ENTRYPOINT /home/gitconsensusservice/start_worker.sh \ No newline at end of file diff --git a/gitconsensusservice/__init__.py b/gitconsensusservice/__init__.py index f1bdac6..32ec5ad 100644 --- a/gitconsensusservice/__init__.py +++ b/gitconsensusservice/__init__.py @@ -11,6 +11,19 @@ with open(os.environ['SETTINGS'], 'r') as infile: app.config.update(yaml.load(infile.read())) + +SETTINGS = [ + 'DEBUG', + 'GITHUB_PRIVATE_KEY', + 'GITHUB_APP_ID', + 'GITHUB_WEBHOOK_SECRET', + 'CELERY_BROKER', + 'PROCESS_INSTALLS_INTERVAL'] +for setting in SETTINGS: + if setting in os.environ: + app.config[setting] = os.environ[setting] + + if 'CELERY_BROKER' in app.config: celery = Celery('gitconsensus', broker=app.config['CELERY_BROKER']) else: diff --git a/gitconsensusservice/cli.py b/gitconsensusservice/cli.py index 13295e1..685c213 100644 --- a/gitconsensusservice/cli.py +++ b/gitconsensusservice/cli.py @@ -41,14 +41,17 @@ def install_repos(install_id): def list_repos(): installs = gh.get_installations() for install_id in installs: - click.echo('Install %s:' % install_id) - installation = get_githubapp_install(install_id) - repos = installation.get_repositories() - for repo in repos: - user, repo = repo.split('/') - repository = installation.get_repository(user, repo) - if repository.rules: - click.echo('\t%s/%s' % (user, repo)) + try: + click.echo('Install %s:' % install_id) + installation = get_githubapp_install(install_id) + repos = installation.get_repositories() + for repo in repos: + user, repo = repo.split('/') + repository = installation.get_repository(user, repo) + if repository.rules: + click.echo('\t%s/%s' % (user, repo)) + except: + click.echo('Unable to process install %s' % install_id) @cli.command(short_help="List details about the current application.") diff --git a/gitconsensusservice/jobs/consensus.py b/gitconsensusservice/jobs/consensus.py index 189e8ae..8a2ecc8 100644 --- a/gitconsensusservice/jobs/consensus.py +++ b/gitconsensusservice/jobs/consensus.py @@ -8,7 +8,11 @@ def process_installs(synchronous=False): installs = gh.get_installations() for install in installs: if synchronous: - process_installation(install, True) + try: + process_installation(install, True) + except Exception as error: + print('Failed to process install %s due to error:' % install) + print(error) else: process_installation.delay(install) @@ -20,7 +24,15 @@ def process_installation(installation_id, synchronous=False): repositories = installation.get_repositories() for repository in repositories: user, repo = repository.split('/') - process_repository(installation_id, user, repo, True) + try: + if synchronous: + process_repository(installation_id, user, repo, True) + else: + process_repository.delay(installation_id, user, repo, False) + except Exception as error: + print('Failed to process %s/%s due to error:' % (user, repo)) + print(error) + @celery.task diff --git a/gitconsensusservice/worker.py b/gitconsensusservice/worker.py index ceb6383..ea0758f 100644 --- a/gitconsensusservice/worker.py +++ b/gitconsensusservice/worker.py @@ -1,2 +1,12 @@ -import gitconsensusservice.jobs.consensus -from gitconsensusservice import celery +from celery.schedules import crontab +from gitconsensusservice.jobs import consensus +from gitconsensusservice import celery, app + + +@celery.on_after_finalize.connect +def setup_periodic_tasks(sender, **kwargs): + print('Root Task - Schedule Installation Jobs') + sender.add_periodic_task( + float(app.config.get('PROCESS_INSTALLS_INTERVAL', 5 * 60.0)), + consensus.process_installs.s(), + name='Root Task - Schedule Installation Jobs') diff --git a/provisioning/etc/gitconsensus/celery b/provisioning/etc/gitconsensus/celery deleted file mode 100644 index 32d4566..0000000 --- a/provisioning/etc/gitconsensus/celery +++ /dev/null @@ -1,29 +0,0 @@ - -# Where does gitconsensus live -CELERY_HOME="/opt/gitconsensus" - -# Name of nodes to start -# here we have a single node -CELERYD_NODES="w1" -# or we could have three nodes: -#CELERYD_NODES="w1 w2 w3" - -# Absolute or relative path to the 'celery' command: -CELERY_BIN="/opt/gitconsensus/venv/bin/celery" - -# App instance to use -# comment out this line if you don't use an app -CELERY_APP="gitconsensusservice.worker:celery" - -# How to call manage.py -CELERYD_MULTI="multi" - -# Extra command-line arguments to the worker -CELERYD_OPTS="--time-limit=120 --concurrency=4" - -# - %n will be replaced with the first part of the nodename. -# - %I will be replaced with the current child process index -# and is important when using the prefork pool to avoid race conditions. -CELERYD_PID_FILE="/var/run/gitconsensus/worker-%n.pid" -CELERYD_LOG_FILE="/var/log/gitconsensus/worker-%n%I.log" -CELERYD_LOG_LEVEL="INFO" diff --git a/provisioning/etc/systemd/system/gitconsensusworker.service b/provisioning/etc/systemd/system/gitconsensusworker.service deleted file mode 100644 index 045c696..0000000 --- a/provisioning/etc/systemd/system/gitconsensusworker.service +++ /dev/null @@ -1,16 +0,0 @@ -[Unit] -Description=GitConsensus Service -After=network.target - -[Service] -Type=forking -User=gitconsensus -Group=gitconsensus -EnvironmentFile=-/etc/gitconsensus/celery -WorkingDirectory=${CELERY_HOME} -ExecStart=/bin/bash -c 'source ${CELERY_HOME}/bin/envvar && ${CELERY_BIN} multi start ${CELERYD_NODES} -A ${CELERY_APP} --pidfile=${CELERYD_PID_FILE} --logfile=${CELERYD_LOG_FILE} --loglevel=${CELERYD_LOG_LEVEL} ${CELERYD_OPTS}' -ExecStop=/bin/bash -c 'source ${CELERY_HOME}/bin/envvar && ${CELERY_BIN} multi stopwait ${CELERYD_NODES} --pidfile=${CELERYD_PID_FILE}' -ExecReload=/bin/bash -c 'source ${CELERY_HOME}/bin/envvar && ${CELERY_BIN} multi restart ${CELERYD_NODES} -A ${CELERY_APP} --pidfile=${CELERYD_PID_FILE} --logfile=${CELERYD_LOG_FILE} --loglevel=${CELERYD_LOG_LEVEL} ${CELERYD_OPTS}' - -[Install] -WantedBy=multi-user.target diff --git a/provisioning/setup.sh b/provisioning/setup.sh deleted file mode 100755 index 7d1e472..0000000 --- a/provisioning/setup.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash - -if [[ -L "${BASH_SOURCE[0]}" ]] -then - DIR="$( cd "$( dirname $( readlink "${BASH_SOURCE[0]}" ) )" && pwd )" -else - DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -fi - -cd $DIR - -cp $DIR/etc/systemd/system/gitconsensusworker.service /etc/systemd/system/gitconsensusworker.service -cp $DIR/etc/gitconsensus/celery /etc/gitconsensus/celery - - -adduser --gecos "" --disabled-login gitconsensus - -mkdir /var/run/gitconsensus -chown gitconsensus:gitconsensus /var/run/gitconsensus -mkdir /var/log/gitconsensus -chown gitconsensus:gitconsensus /var/log/gitconsensus - -echo "d /var/run/celery 0755 celery celery -" >> /etc/tmpfiles.d/celery.conf -echo "d /var/log/celery 0755 celery celery - " >> /etc/tmpfiles.d/celery.conf - -systemctl daemon-reload diff --git a/requirements.txt b/requirements.txt index b45fe9a..fc3b1c5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,30 +1,16 @@ -amqp==2.2.2 -asn1crypto==0.24.0 -billiard==3.5.0.3 -celery==4.1.0 -certifi==2018.1.18 -cffi==1.11.5 -chardet==3.0.4 -click==6.7 -cryptography==2.2.2 -Flask==0.12.2 -gitconsensus==0.7.0 -github3.py==1.0.2 -github3apps.py==0.1.2 -idna==2.6 -itsdangerous==0.24 -Jinja2==2.10 -kombu==4.1.0 -MarkupSafe==1.0 -pycparser==2.18 -PyJWT==1.6.1 -python-dateutil==2.7.2 -pytz==2018.3 -PyYAML==3.12 -requests==2.18.4 -six==1.11.0 -uritemplate==3.0.0 -uritemplate.py==3.0.2 -urllib3==1.22 -vine==1.1.4 -Werkzeug==0.14.1 + +# Frameworks +celery==4.3.0 +click==7.0 +Flask==1.0.2 + + +# Git +gitconsensus==0.8.0 +github3.py==1.3.0 +github3apps.py==0.2.0 + + +# Libraries +PyYAML==5.1 +requests==2.21.0 diff --git a/setup.py b/setup.py index 44ce6e0..fcbf4b5 100644 --- a/setup.py +++ b/setup.py @@ -50,7 +50,7 @@ 'Flask>=0.12.2', 'gitconsensus>=0.7,<0.8' 'github3.py>=1,<2', - 'github3apps.py>=0.1.2', + 'github3apps.py>=0.1.3,<0.2', 'pyjwt>=1.5.3,<2', 'PyYAML>=3.12,<3.13', 'requests>=2.18.0,<2.19',