diff --git a/mod_ci/controllers.py b/mod_ci/controllers.py index 3aa84688..216f917e 100755 --- a/mod_ci/controllers.py +++ b/mod_ci/controllers.py @@ -6,7 +6,6 @@ import os import shutil import sys -from multiprocessing import Process from typing import Any import requests @@ -40,9 +39,6 @@ from mod_test.models import (Fork, Test, TestPlatform, TestProgress, TestResult, TestResultFile, TestStatus, TestType) -if sys.platform.startswith("linux"): - import libvirt - mod_ci = Blueprint('ci', __name__) @@ -72,347 +68,6 @@ def before_app_request() -> None: g.menu_entries['config'] = config_entries -def start_platforms(db, repository, delay=None, platform=None) -> None: - """ - Start new test on both platforms in parallel. - - We use multiprocessing module which bypasses Python GIL to make use of multiple cores of the processor. - """ - from run import app, config, log - - with app.app_context(): - from flask import current_app - if platform is None or platform == TestPlatform.linux: - linux_kvm_name = config.get('KVM_LINUX_NAME', '') - log.info('Define process to run Linux VM') - linux_process = Process(target=kvm_processor, args=(current_app._get_current_object(), db, linux_kvm_name, - TestPlatform.linux, repository, delay,)) - linux_process.start() - log.info('Linux VM process kicked off') - - if platform is None or platform == TestPlatform.windows: - win_kvm_name = config.get('KVM_WINDOWS_NAME', '') - log.info('Define process to run Windows VM') - windows_process = Process(target=kvm_processor, args=(current_app._get_current_object(), db, win_kvm_name, - TestPlatform.windows, repository, delay,)) - windows_process.start() - log.info('Windows VM process kicked off') - - -def kvm_processor(app, db, kvm_name, platform, repository, delay) -> None: - """ - Check whether there is no already running same kvm. - - Checks whether machine is in maintenance mode or not - Launch kvm if not used by any other test - Creates testing xml files to test the change in main repo. - Creates clone with separate branch and merge pr into it. - - :param app: The Flask app - :type app: Flask - :param db: database connection - :type db: sqlalchemy.orm.scoped_session - :param kvm_name: name for the kvm - :type kvm_name: str - :param platform: operating system - :type platform: str - :param repository: repository to run tests on - :type repository: str - :param delay: time delay after which to start kvm processor - :type delay: int - """ - from run import config, get_github_config, log - - github_config = get_github_config(config) - - log.info(f"[{platform}] Running kvm_processor") - if kvm_name == "": - log.critical(f'[{platform}] KVM name is empty!') - return - - if delay is not None: - import time - log.debug(f'[{platform}] Sleeping for {delay} seconds') - time.sleep(delay) - - maintenance_mode = MaintenanceMode.query.filter(MaintenanceMode.platform == platform).first() - if maintenance_mode is not None and maintenance_mode.disabled: - log.debug(f'[{platform}] In maintenance mode! Waiting...') - return - - conn = libvirt.open("qemu:///system") - if conn is None: - log.critical(f"[{platform}] Connection to libvirt failed!") - return - - try: - vm = conn.lookupByName(kvm_name) - except libvirt.libvirtError: - log.critical(f"[{platform}] No VM named {kvm_name} found!") - return - - vm_info = vm.info() - if vm_info[0] != libvirt.VIR_DOMAIN_SHUTOFF: - # Running, check expiry and compare to runtime - status = Kvm.query.filter(Kvm.name == kvm_name).first() - max_runtime = config.get("KVM_MAX_RUNTIME", 120) - if status is not None: - if datetime.datetime.now() - status.timestamp >= datetime.timedelta(minutes=max_runtime): - test_progress = TestProgress(status.test.id, TestStatus.canceled, 'Runtime exceeded') - db.add(test_progress) - db.delete(status) - db.commit() - - if vm.destroy() == -1: - log.critical(f"[{platform}] Failed to shut down {kvm_name}") - return - else: - log.info(f"[{platform}] Current job not expired yet.") - return - else: - log.warn(f"[{platform}] No task, but VM is running! Hard reset necessary") - if vm.destroy() == -1: - log.critical(f"[{platform}] Failed to shut down {kvm_name}") - return - - # Check if there's no KVM status left - status = Kvm.query.filter(Kvm.name == kvm_name).first() - if status is not None: - log.warn(f"[{platform}] KVM is powered off, but test {status.test.id} still present, deleting entry") - db.delete(status) - db.commit() - - # Get oldest test for this platform - finished_tests = db.query(TestProgress.test_id).filter( - TestProgress.status.in_([TestStatus.canceled, TestStatus.completed]) - ).subquery() - fork_location = f"%/{github_config['repository_owner']}/{github_config['repository']}.git" - fork = Fork.query.filter(Fork.github.like(fork_location)).first() - test = Test.query.filter( - Test.id.notin_(finished_tests), Test.platform == platform, Test.fork_id == fork.id - ).order_by(Test.id.asc()).first() - - if test is None: - test = Test.query.filter(Test.id.notin_(finished_tests), Test.platform == platform).order_by( - Test.id.asc()).first() - - if test is None: - log.info(f'[{platform}] No more tests to run, returning') - return - - if test.test_type == TestType.pull_request and test.pr_nr == 0: - log.warn(f'[{platform}] Test {test.id} is invalid, deleting') - db.delete(test) - db.commit() - return - - # Reset to snapshot - if vm.hasCurrentSnapshot() != 1: - log.critical(f"[{platform}] VM {kvm_name} has no current snapshot set!") - return - - snapshot = vm.snapshotCurrent() - if vm.revertToSnapshot(snapshot) == -1: - log.critical(f"[{platform}] Failed to revert to {snapshot.getName()} for {kvm_name}") - return - - log.info(f"[{platform}] Reverted to {snapshot.getName()} for {kvm_name}") - log.debug(f'[{platform}] Starting test {test.id}') - status = Kvm(kvm_name, test.id) - # Prepare data - # 0) Write url to file - with app.app_context(): - full_url = url_for('ci.progress_reporter', test_id=test.id, token=test.token, _external=True, _scheme="https") - - file_path = os.path.join(config.get('SAMPLE_REPOSITORY', ''), 'vm_data', kvm_name, 'reportURL') - - with open(file_path, 'w') as f: - f.write(full_url) - - # 1) Generate test files - base_folder = os.path.join(config.get('SAMPLE_REPOSITORY', ''), 'vm_data', kvm_name, 'ci-tests') - categories = Category.query.order_by(Category.id.desc()).all() - commit_name = 'fetch_commit_' + platform.value - commit_hash = GeneralData.query.filter(GeneralData.key == commit_name).first().value - last_commit = Test.query.filter(and_(Test.commit == commit_hash, Test.platform == platform)).first() - - log.debug(f"[{platform}] We will compare against the results of test {last_commit.id}") - - regression_ids = test.get_customized_regressiontests() - - # Init collection file - multi_test = etree.Element('multitest') - for category in categories: - # Skip categories without tests - if len(category.regression_tests) == 0: - continue - # Create XML file for test - file_name = f'{category.name}.xml' - single_test = etree.Element('tests') - should_write_xml = False - for regression_test in category.regression_tests: - if regression_test.id not in regression_ids: - log.debug(f'Skipping RT #{regression_test.id} ({category.name}) as not in scope') - continue - should_write_xml = True - entry = etree.SubElement(single_test, 'entry', id=str(regression_test.id)) - command = etree.SubElement(entry, 'command') - command.text = regression_test.command - input_node = etree.SubElement(entry, 'input', type=regression_test.input_type.value) - # Need a path that is relative to the folder we provide inside the CI environment. - input_node.text = regression_test.sample.filename - output_node = etree.SubElement(entry, 'output') - output_node.text = regression_test.output_type.value - compare = etree.SubElement(entry, 'compare') - last_files = TestResultFile.query.filter(and_( - TestResultFile.test_id == last_commit.id, - TestResultFile.regression_test_id == regression_test.id - )).subquery() - - for output_file in regression_test.output_files: - ignore_file = str(output_file.ignore).lower() - file_node = etree.SubElement(compare, 'file', ignore=ignore_file, id=str(output_file.id)) - last_commit_files = db.query(last_files.c.got).filter(and_( - last_files.c.regression_test_output_id == output_file.id, - last_files.c.got.isnot(None) - )).first() - correct = etree.SubElement(file_node, 'correct') - # Need a path that is relative to the folder we provide inside the CI environment. - if last_commit_files is None: - log.debug(f"Selecting original file for RT #{regression_test.id} ({category.name})") - correct.text = output_file.filename_correct - else: - correct.text = output_file.create_correct_filename(last_commit_files[0]) - - expected = etree.SubElement(file_node, 'expected') - expected.text = output_file.filename_expected(regression_test.sample.sha) - if not should_write_xml: - continue - save_xml_to_file(single_test, base_folder, file_name) - # Append to collection file - test_file = etree.SubElement(multi_test, 'testfile') - location = etree.SubElement(test_file, 'location') - location.text = file_name - - save_xml_to_file(multi_test, base_folder, 'TestAll.xml') - - # 2) Create git repo clone and merge PR into it (if necessary) - try: - repo = Repo(os.path.join(config.get('SAMPLE_REPOSITORY', ''), 'vm_data', kvm_name, 'unsafe-ccextractor')) - except InvalidGitRepositoryError: - log.critical(f"[{platform}] Could not open CCExtractor's repository copy!") - return - - # Return to master - repo.heads.master.checkout(True) - # Update repository from upstream - try: - github_url = test.fork.github - if is_main_repo(github_url): - origin = repo.remote('origin') - else: - fork_id = test.fork.id - remote = f'fork_{fork_id}' - if remote in [remote.name for remote in repo.remotes]: - origin = repo.remote(remote) - else: - origin = repo.create_remote(remote, url=github_url) - except ValueError: - log.critical(f"[{platform}] Origin remote doesn't exist!") - return - - fetch_info = origin.fetch() - if len(fetch_info) == 0: - log.info(f'[{platform}] Fetch from remote returned no new data...') - # Checkout to Remote Master - repo.git.checkout(origin.refs.master) - # Pull code (finally) - pull_info = origin.pull('master') - if len(pull_info) == 0: - log.info(f"[{platform}] Pull from remote returned no new data...") - elif pull_info[0].flags > 128: - log.critical(f"[{platform}] Did not pull any information from remote: {pull_info[0].flags}!") - return - - ci_branch = 'CI_Branch' - # Delete the test branch if it exists, and recreate - try: - repo.delete_head(ci_branch, force=True) - except GitCommandError: - log.info(f"[{platform}] Could not delete CI_Branch head") - - # Remove possible left rebase-apply directory - try: - shutil.rmtree(os.path.join(config.get('SAMPLE_REPOSITORY', ''), 'unsafe-ccextractor', '.git', 'rebase-apply')) - except OSError: - log.info(f"[{platform}] Could not delete rebase-apply") - # If PR, merge, otherwise reset to commit - if test.test_type == TestType.pull_request: - # Fetch PR (stored under origin/pull//head) - pull_info = origin.fetch(f'pull/{test.pr_nr}/head:{ci_branch}') - if len(pull_info) == 0: - log.warn(f"[{platform}] Did not pull any information from remote PR!") - elif pull_info[0].flags > 128: - log.critical(f"[{platform}] Did not pull any information from remote PR: {pull_info[0].flags}!") - return - - try: - test_branch = repo.heads[ci_branch] - except IndexError: - log.critical(f'{ci_branch} does not exist') - return - - test_branch.checkout(True) - - try: - pull = repository.pulls(f'{test.pr_nr}').get() - except ApiError as a: - log.error(f'Got an exception while fetching the PR payload! Message: {a.message}') - return - if pull['mergeable'] is False: - progress = TestProgress(test.id, TestStatus.canceled, "Commit could not be merged", datetime.datetime.now()) - db.add(progress) - db.commit() - try: - with app.app_context(): - repository.statuses(test.commit).post( - state=Status.FAILURE, - description="Tests canceled due to merge conflict", - context=f"CI - {test.platform.value}", - target_url=url_for('test.by_id', test_id=test.id, _external=True) - ) - except ApiError as a: - log.error(f'Got an exception while posting to GitHub! Message: {a.message}') - return - - # Merge on master if no conflict - repo.git.merge('master') - - else: - test_branch = repo.create_head(ci_branch, origin.refs.master) - test_branch.checkout(True) - try: - repo.head.reset(test.commit, working_tree=True) - except GitCommandError: - log.warn(f"[{platform}] Commit {test.commit} for test {test.id} does not exist!") - return - - # Power on machine - try: - vm.create() - db.add(status) - db.commit() - except libvirt.libvirtError as e: - log.critical(f"[{platform}] Failed to launch VM {kvm_name}") - log.critical(f"Information about failure: code: {e.get_error_code()}, domain: {e.get_error_domain()}, " - f"level: {e.get_error_level()}, message: {e.get_error_message()}") - except IntegrityError: - log.warn(f"[{platform}] Duplicate entry for {test.id}") - - # Close connection to libvirt - conn.close() - - def save_xml_to_file(xml_node, folder_name, file_name) -> None: """ Save the given XML node to a file in a certain folder. @@ -459,6 +114,7 @@ def queue_test(db, gh_commit, commit, test_type, branch="master", pr_nr=0) -> No log.debug('pull request test type detected') branch = "pull_request" + # TODO: launch GCP instance directly linux_test = Test(TestPlatform.linux, test_type, fork.id, branch, commit, pr_nr) db.add(linux_test) windows_test = Test(TestPlatform.windows, test_type, fork.id, branch, commit, pr_nr) @@ -626,23 +282,6 @@ def start_ci(): queue_test(g.db, github_status, commit_hash, TestType.pull_request, pr_nr=pr_nr) - elif payload['action'] == 'closed': - g.log.debug('PR was closed, no after hash available') - # Cancel running queue - tests = Test.query.filter(Test.pr_nr == pr_nr).all() - for test in tests: - # Add cancelled status only if the test hasn't started yet - if len(test.progress) > 0: - continue - progress = TestProgress(test.id, TestStatus.canceled, "PR closed", datetime.datetime.now()) - g.db.add(progress) - repository.statuses(test.commit).post( - state=Status.FAILURE, - description="Tests canceled", - context=f"CI - {test.platform.value}", - target_url=url_for('test.by_id', test_id=test.id, _external=True) - ) - elif event == "issues": g.log.debug('issues event detected') diff --git a/mod_ci/cron.py b/mod_ci/cron.py deleted file mode 100755 index 9c0a0aed..00000000 --- a/mod_ci/cron.py +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/python -"""handle cron logic for CI platform.""" - -import sys -from os import path - -# Need to append server root path to ensure we can import the necessary files. -sys.path.append(path.dirname(path.dirname(path.abspath(__file__)))) - - -def cron(testing=False): - """Script to run from cron for Sampleplatform.""" - from flask import current_app - from github import GitHub - - from database import create_session - from mod_ci.controllers import TestPlatform, kvm_processor, start_platforms - from run import config, log - - log.info('Run the cron for kicking off CI platform(s).') - # Create session - db = create_session(config['DATABASE_URI']) - gh = GitHub(access_token=config['GITHUB_TOKEN']) - repository = gh.repos(config['GITHUB_OWNER'])(config['GITHUB_REPOSITORY']) - - if testing is True: - kvm_processor(current_app._get_current_object(), db, config.get('KVM_LINUX_NAME', ''), TestPlatform.linux, - repository, None) - else: - start_platforms(db, repository) - - -cron() diff --git a/requirements.txt b/requirements.txt index 95994fb2..e047ac60 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,7 +13,6 @@ xmltodict==0.12.0 lxml==4.6.3 pytz==2021.1 tzlocal==2.1 -libvirt-python==7.1.0; sys_platform != 'win32' markdown2==2.4.1 flask-migrate==3.1.0 flask-script==2.0.6 diff --git a/tests/test_ci/TestControllers.py b/tests/test_ci/TestControllers.py index 3af6577f..356f1605 100644 --- a/tests/test_ci/TestControllers.py +++ b/tests/test_ci/TestControllers.py @@ -7,7 +7,7 @@ from werkzeug.datastructures import Headers from mod_auth.models import Role -from mod_ci.controllers import get_info_for_pr_comment, start_platforms +from mod_ci.controllers import get_info_for_pr_comment from mod_ci.models import BlockedUsers from mod_customized.models import CustomizedTest from mod_home.models import CCExtractorVersion, GeneralData @@ -110,82 +110,6 @@ def test_comment_info_handles_invalid_variants_correctly(self): for stat in stats: self.assertEqual(stat.success, None) - @mock.patch('mod_ci.controllers.Process') - @mock.patch('run.log') - def test_start_platform_none_specified(self, mock_log, mock_process): - """Test that both platforms run with no platform value is passed.""" - start_platforms(mock.ANY, mock.ANY) - - self.assertEqual(2, mock_process.call_count) - self.assertEqual(4, mock_log.info.call_count) - - @mock.patch('mod_ci.controllers.Process') - @mock.patch('run.log') - def test_start_platform_linux_specified(self, mock_log, mock_process): - """Test that only linux platform runs.""" - start_platforms(mock.ANY, mock.ANY, platform=TestPlatform.linux) - - self.assertEqual(1, mock_process.call_count) - self.assertEqual(2, mock_log.info.call_count) - mock_log.info.assert_called_with("Linux VM process kicked off") - - @mock.patch('mod_ci.controllers.Process') - @mock.patch('run.log') - def test_start_platform_windows_specified(self, mock_log, mock_process): - """Test that only windows platform runs.""" - start_platforms(mock.ANY, mock.ANY, platform=TestPlatform.windows) - - self.assertEqual(1, mock_process.call_count) - self.assertEqual(2, mock_log.info.call_count) - mock_log.info.assert_called_with("Windows VM process kicked off") - - @mock.patch('run.log') - def test_kvm_processor_empty_kvm_name(self, mock_log): - """Test that kvm processor fails with empty kvm name.""" - from mod_ci.controllers import kvm_processor - - resp = kvm_processor(mock.ANY, mock.ANY, "", mock.ANY, mock.ANY, mock.ANY) - - self.assertIsNone(resp) - mock_log.info.assert_called_once() - mock_log.critical.assert_called_once() - - @mock.patch('run.log') - @mock.patch('mod_ci.controllers.MaintenanceMode') - def test_kvm_processor_maintenance_mode(self, mock_maintenance, mock_log): - """Test that kvm processor does not run when in mentainenace.""" - from mod_ci.controllers import kvm_processor - - class MockMaintence: - def __init__(self): - self.disabled = True - - mock_maintenance.query.filter.return_value.first.return_value = MockMaintence() - - resp = kvm_processor(mock.ANY, mock.ANY, "test", mock.ANY, mock.ANY, 1) - - self.assertIsNone(resp) - mock_log.info.assert_called_once() - mock_log.critical.assert_not_called() - self.assertEqual(mock_log.debug.call_count, 2) - - @mock.patch('mod_ci.controllers.libvirt') - @mock.patch('run.log') - @mock.patch('mod_ci.controllers.MaintenanceMode') - def test_kvm_processor_conn_fail(self, mock_maintenance, mock_log, mock_libvirt): - """Test that kvm processor logs critically when conn cannot be established.""" - from mod_ci.controllers import kvm_processor - - mock_libvirt.open.return_value = None - mock_maintenance.query.filter.return_value.first.return_value = None - - resp = kvm_processor(mock.ANY, mock.ANY, "test", mock.ANY, mock.ANY, 1) - - self.assertIsNone(resp) - mock_log.info.assert_called_once() - mock_log.critical.assert_called_once() - self.assertEqual(mock_log.debug.call_count, 1) - @mock.patch('mod_ci.controllers.GeneralData') @mock.patch('mod_ci.controllers.g') def test_set_avg_time_first(self, mock_g, mock_gd): @@ -275,113 +199,6 @@ def test_check_main_repo_returns_in_false_url(self): assert is_main_repo('random_user/random_repo') is False assert is_main_repo('test_owner/test_repo') is True - @mock.patch('github.GitHub') - @mock.patch('git.Repo') - @mock.patch('libvirt.open') - @mock.patch('shutil.rmtree') - @mock.patch('mod_ci.controllers.open') - @mock.patch('lxml.etree') - def test_customize_tests_run_on_fork_if_no_remote(self, mock_etree, mock_open, - mock_rmtree, mock_libvirt, mock_repo, mock_git): - """Test customize tests running on the fork without remote.""" - self.create_user_with_role( - self.user.name, self.user.email, self.user.password, Role.tester) - self.create_forktest("own-fork-commit", TestPlatform.linux) - import mod_ci.controllers - import mod_ci.cron - reload(mod_ci.cron) - reload(mod_ci.controllers) - from mod_ci.cron import cron - conn = mock_libvirt() - vm = conn.lookupByName() - import libvirt - - # mocking the libvirt kvm to shut down - vm.info.return_value = [libvirt.VIR_DOMAIN_SHUTOFF] - # Setting current snapshot of libvirt - vm.hasCurrentSnapshot.return_value = 1 - repo = mock_repo() - origin = repo.create_remote() - from collections import namedtuple - GitPullInfo = namedtuple('GitPullInfo', 'flags') - pull_info = GitPullInfo(flags=0) - origin.pull.return_value = [pull_info] - cron(testing=True) - fork_url = f"https://github.com/{self.user.name}/{g.github['repository']}.git" - repo.create_remote.assert_called_with("fork_2", url=fork_url) - repo.create_head.assert_called_with("CI_Branch", origin.refs.master) - - @mock.patch('github.GitHub') - @mock.patch('git.Repo') - @mock.patch('libvirt.open') - @mock.patch('shutil.rmtree') - @mock.patch('mod_ci.controllers.open') - @mock.patch('lxml.etree') - def test_customize_tests_run_on_fork_if_remote_exist(self, mock_etree, mock_open, - mock_rmtree, mock_libvirt, mock_repo, mock_git): - """Test customize tests running on the fork with remote.""" - self.create_user_with_role(self.user.name, self.user.email, self.user.password, Role.tester) - self.create_forktest("own-fork-commit", TestPlatform.linux) - import mod_ci.controllers - import mod_ci.cron - reload(mod_ci.cron) - reload(mod_ci.controllers) - from mod_ci.cron import cron - conn = mock_libvirt() - vm = conn.lookupByName() - import libvirt - - # mocking the libvirt kvm to shut down - vm.info.return_value = [libvirt.VIR_DOMAIN_SHUTOFF] - # Setting current snapshot of libvirt - vm.hasCurrentSnapshot.return_value = 1 - repo = mock_repo() - origin = repo.remote() - from collections import namedtuple - Remotes = namedtuple('Remotes', 'name') - repo.remotes = [Remotes(name='fork_2')] - GitPullInfo = namedtuple('GitPullInfo', 'flags') - pull_info = GitPullInfo(flags=0) - origin.pull.return_value = [pull_info] - cron(testing=True) - repo.remote.assert_called_with('fork_2') - - @mock.patch('github.GitHub') - @mock.patch('git.Repo') - @mock.patch('libvirt.open') - @mock.patch('shutil.rmtree') - @mock.patch('mod_ci.controllers.open') - @mock.patch('lxml.etree') - def test_customize_tests_run_on_selected_regression_tests(self, mock_etree, mock_open, - mock_rmtree, mock_libvirt, mock_repo, mock_git): - """Test customize tests running on the selected regression tests.""" - self.create_user_with_role( - self.user.name, self.user.email, self.user.password, Role.tester) - self.create_forktest("own-fork-commit", TestPlatform.linux, regression_tests=[2]) - import mod_ci.controllers - import mod_ci.cron - reload(mod_ci.cron) - reload(mod_ci.controllers) - from mod_ci.cron import cron - conn = mock_libvirt() - vm = conn.lookupByName() - import libvirt - vm.info.return_value = [libvirt.VIR_DOMAIN_SHUTOFF] - vm.hasCurrentSnapshot.return_value = 1 - repo = mock_repo() - origin = repo.remote() - from collections import namedtuple - Remotes = namedtuple('Remotes', 'name') - repo.remotes = [Remotes(name='fork_2')] - GitPullInfo = namedtuple('GitPullInfo', 'flags') - pull_info = GitPullInfo(flags=0) - origin.pull.return_value = [pull_info] - single_test = mock_etree.Element('tests') - mock_etree.Element.return_value = single_test - cron(testing=True) - mock_etree.SubElement.assert_any_call(single_test, 'entry', id=str(2)) - assert (single_test, 'entry', str(1)) not in mock_etree.call_args_list - def test_customizedtest_added_to_queue(self): """Test queue with a customized test addition.""" regression_test = RegressionTest.query.filter(RegressionTest.id == 1).first()