From aec53e0ecbf8590bc219158e7716129405b1d808 Mon Sep 17 00:00:00 2001 From: Anders Lantz Date: Wed, 27 Sep 2023 09:32:54 +0200 Subject: [PATCH 1/2] Major restructure of project in order to use multiple modules, needed to speed up resolving dependencies. GroovyBuilderDeployment.groovy * Now has a couple of new methods, still needs some testing * buildAndHost() * hostM2Volume() * buildLocalSources() Container.groovy * added prepareVolumeMount(), getHomePath() * createTar() now has a ignorePaths parameter and supports long file names * ContainerCallback() now debug logs input --- .../buildScripts/createStandalonePom.groovy | 11 +- Environments/AWS/variables.tf | 4 +- devstack-ext/bitbucketinstancemanager/pom.xml | 41 +++ devstack/devstack.iml | 8 + devstack/pom.xml | 237 +++++++++++++ .../devstack/container/Container.groovy | 105 ++++-- .../container/impl/AlpineContainer.groovy | 0 .../container/impl/BitbucketContainer.groovy | 0 .../container/impl/DoodContainer.groovy | 0 .../container/impl/GroovyContainer.groovy | 319 ++++++++++++++++++ .../container/impl/GroovyDoodContainer.groovy | 0 .../impl/HarborManagerContainer.groovy | 0 .../container/impl/JenkinsContainer.groovy | 0 .../container/impl/JsmContainer.groovy | 0 .../container/impl/NginxContainer.groovy | 0 .../container/impl/UbuntuContainer.groovy | 0 .../devstack/deployment/Deployment.groovy | 0 .../impl/BitbucketH2Deployment.groovy | 0 .../impl/GroovyBuilderDeployment.groovy | 165 +++++++++ .../deployment/impl/HarborDeployment.groovy | 0 .../impl/JenkinsAndHarborDeployment.groovy | 0 .../deployment/impl/JenkinsDeployment.groovy | 0 .../impl/JsmAndBitbucketH2Deployment.groovy | 0 .../deployment/impl/JsmH2Deployment.groovy | 0 .../deployment/impl/NginxFileServer.groovy | 1 + .../eficode/devstack/simplelogger.properties | 0 .../eficode/devstack/util/ImageBuilder.groovy | 0 .../eficode/devstack/util/TimeMachine.groovy | 0 .../jira/scripts/CreateBitbucketLink.groovy | 2 +- .../src}/main/simplelogger.properties | 0 .../com/eficode/devstack/DevStackSpec.groovy | 6 +- .../devstack/container/ContainerTest.groovy | 0 .../impl/BitbucketContainerTest.groovy | 0 .../container/impl/DoodContainerTest.groovy | 2 +- .../container/impl/GroovyContainerTest.groovy | 57 ++++ .../impl/JenkinsContainerTest.groovy | 13 +- .../container/impl/JsmContainerTest.groovy | 54 +-- .../container/impl/NginxContainerTest.groovy | 12 +- .../impl/BitbucketH2DeploymentTest.groovy | 7 +- .../impl/HarborDeploymentTest.groovy | 4 +- .../JenkinsAndHarborDeploymentTest.groovy | 2 +- .../JsmAndBitbucketH2DeploymentTest.groovy | 2 +- .../impl/JsmH2DeploymentTest.groovy | 2 +- .../devstack/examples/ExamplesTest.groovy | 2 +- .../src}/test/groovy/simplelogger.properties | 0 pom.xml | 219 +----------- .../container/impl/GroovyContainer.groovy | 53 --- .../devstack/util/DockerClientDS.groovy | 80 ----- 48 files changed, 978 insertions(+), 430 deletions(-) create mode 100644 devstack-ext/bitbucketinstancemanager/pom.xml create mode 100644 devstack/devstack.iml create mode 100644 devstack/pom.xml rename {src => devstack/src}/main/groovy/com/eficode/devstack/container/Container.groovy (89%) rename {src => devstack/src}/main/groovy/com/eficode/devstack/container/impl/AlpineContainer.groovy (100%) rename {src => devstack/src}/main/groovy/com/eficode/devstack/container/impl/BitbucketContainer.groovy (100%) rename {src => devstack/src}/main/groovy/com/eficode/devstack/container/impl/DoodContainer.groovy (100%) create mode 100644 devstack/src/main/groovy/com/eficode/devstack/container/impl/GroovyContainer.groovy rename {src => devstack/src}/main/groovy/com/eficode/devstack/container/impl/GroovyDoodContainer.groovy (100%) rename {src => devstack/src}/main/groovy/com/eficode/devstack/container/impl/HarborManagerContainer.groovy (100%) rename {src => devstack/src}/main/groovy/com/eficode/devstack/container/impl/JenkinsContainer.groovy (100%) rename {src => devstack/src}/main/groovy/com/eficode/devstack/container/impl/JsmContainer.groovy (100%) rename {src => devstack/src}/main/groovy/com/eficode/devstack/container/impl/NginxContainer.groovy (100%) rename {src => devstack/src}/main/groovy/com/eficode/devstack/container/impl/UbuntuContainer.groovy (100%) rename {src => devstack/src}/main/groovy/com/eficode/devstack/deployment/Deployment.groovy (100%) rename {src => devstack/src}/main/groovy/com/eficode/devstack/deployment/impl/BitbucketH2Deployment.groovy (100%) create mode 100644 devstack/src/main/groovy/com/eficode/devstack/deployment/impl/GroovyBuilderDeployment.groovy rename {src => devstack/src}/main/groovy/com/eficode/devstack/deployment/impl/HarborDeployment.groovy (100%) rename {src => devstack/src}/main/groovy/com/eficode/devstack/deployment/impl/JenkinsAndHarborDeployment.groovy (100%) rename {src => devstack/src}/main/groovy/com/eficode/devstack/deployment/impl/JenkinsDeployment.groovy (100%) rename {src => devstack/src}/main/groovy/com/eficode/devstack/deployment/impl/JsmAndBitbucketH2Deployment.groovy (100%) rename {src => devstack/src}/main/groovy/com/eficode/devstack/deployment/impl/JsmH2Deployment.groovy (100%) rename {src => devstack/src}/main/groovy/com/eficode/devstack/deployment/impl/NginxFileServer.groovy (97%) rename {src => devstack/src}/main/groovy/com/eficode/devstack/simplelogger.properties (100%) rename {src => devstack/src}/main/groovy/com/eficode/devstack/util/ImageBuilder.groovy (100%) rename {src => devstack/src}/main/groovy/com/eficode/devstack/util/TimeMachine.groovy (100%) rename {src => devstack/src}/main/resources/com/eficode/devstack/deployment/jira/scripts/CreateBitbucketLink.groovy (93%) rename {src => devstack/src}/main/simplelogger.properties (100%) rename {src => devstack/src}/test/groovy/com/eficode/devstack/DevStackSpec.groovy (97%) rename {src => devstack/src}/test/groovy/com/eficode/devstack/container/ContainerTest.groovy (100%) rename {src => devstack/src}/test/groovy/com/eficode/devstack/container/impl/BitbucketContainerTest.groovy (100%) rename {src => devstack/src}/test/groovy/com/eficode/devstack/container/impl/DoodContainerTest.groovy (95%) create mode 100644 devstack/src/test/groovy/com/eficode/devstack/container/impl/GroovyContainerTest.groovy rename {src => devstack/src}/test/groovy/com/eficode/devstack/container/impl/JenkinsContainerTest.groovy (82%) rename {src => devstack/src}/test/groovy/com/eficode/devstack/container/impl/JsmContainerTest.groovy (78%) rename {src => devstack/src}/test/groovy/com/eficode/devstack/container/impl/NginxContainerTest.groovy (92%) rename {src => devstack/src}/test/groovy/com/eficode/devstack/deployment/impl/BitbucketH2DeploymentTest.groovy (86%) rename {src => devstack/src}/test/groovy/com/eficode/devstack/deployment/impl/HarborDeploymentTest.groovy (96%) rename {src => devstack/src}/test/groovy/com/eficode/devstack/deployment/impl/JenkinsAndHarborDeploymentTest.groovy (96%) rename {src => devstack/src}/test/groovy/com/eficode/devstack/deployment/impl/JsmAndBitbucketH2DeploymentTest.groovy (96%) rename {src => devstack/src}/test/groovy/com/eficode/devstack/deployment/impl/JsmH2DeploymentTest.groovy (96%) rename {src => devstack/src}/test/groovy/com/eficode/devstack/examples/ExamplesTest.groovy (98%) rename {src => devstack/src}/test/groovy/simplelogger.properties (100%) delete mode 100644 src/main/groovy/com/eficode/devstack/container/impl/GroovyContainer.groovy delete mode 100644 src/main/groovy/com/eficode/devstack/util/DockerClientDS.groovy diff --git a/.github/buildScripts/createStandalonePom.groovy b/.github/buildScripts/createStandalonePom.groovy index 910c1e0..86a987f 100644 --- a/.github/buildScripts/createStandalonePom.groovy +++ b/.github/buildScripts/createStandalonePom.groovy @@ -3,13 +3,18 @@ * additional build steps for shading based on shadingConf.xml */ String projectBasePath = project.basedir +println("Base dir:" + projectBasePath) File origPom = new File(projectBasePath + "/pom.xml") -File shadingConf = new File(projectBasePath + "/.github/buildScripts/shadingConf.xml") +File shadingConf = new File(origPom.parentFile.parentFile.absolutePath + "/.github/buildScripts/shadingConf.xml") + String newPomBody = origPom.text.replace("", "\n" + shadingConf.text) newPomBody = newPomBody.replaceFirst("devstack<\\/artifactId>", "devstack-standalone<\\/artifactId>") -File standalonePom = new File(projectBasePath + "/pom-standalone.xml") +File standalonePom = new File(projectBasePath + "/devstack-standalone" + "/pom-standalone.xml") +standalonePom.mkdirs() standalonePom.createNewFile() -standalonePom.text = newPomBody \ No newline at end of file +standalonePom.text = newPomBody + +println("Created:" + standalonePom.absolutePath) \ No newline at end of file diff --git a/Environments/AWS/variables.tf b/Environments/AWS/variables.tf index 2ccd71b..f333b1f 100644 --- a/Environments/AWS/variables.tf +++ b/Environments/AWS/variables.tf @@ -39,7 +39,7 @@ variable "ssh-public-key-local-path" { variable "trusted-external-ips" { - description = "These IPs will have acces to the exposed ports" + description = "These IPs will have access to the exposed ports" type = set(string) default = ["1.2.3.4/32"] @@ -58,7 +58,7 @@ variable "ec2-instance-type" { } variable "ingress_rules_from_trusted" { - description = "This will expose the coresponding ports to the internet, but limited to trusted-external-ips" + description = "This will expose the corresponding ports to the internet, but limited to trusted-external-ips" type = list(object({ port = number protocol = string diff --git a/devstack-ext/bitbucketinstancemanager/pom.xml b/devstack-ext/bitbucketinstancemanager/pom.xml new file mode 100644 index 0000000..66955f5 --- /dev/null +++ b/devstack-ext/bitbucketinstancemanager/pom.xml @@ -0,0 +1,41 @@ + + + 4.0.0 + + com.eficode + bitbucketinstancemanager + 1.0 + pom + + com.eficode + devstack-parent + 1.0 + + + + 0.0.3-SNAPSHOT-groovy-3.0 + 11 + 11 + UTF-8 + + + + com.eficode.atlassian + bitbucketinstancemanager + ${bitbucketinstancemanager.version} + standalone + + + + + + eficode-github-BitbucketInstanceManagerRest + https://github.com/eficode/BitbucketInstanceManagerRest/raw/packages/repository/ + + + + + + \ No newline at end of file diff --git a/devstack/devstack.iml b/devstack/devstack.iml new file mode 100644 index 0000000..6930588 --- /dev/null +++ b/devstack/devstack.iml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/devstack/pom.xml b/devstack/pom.xml new file mode 100644 index 0000000..141aad3 --- /dev/null +++ b/devstack/pom.xml @@ -0,0 +1,237 @@ + + + 4.0.0 + + + com.eficode + devstack-parent + 1.0 + + + devstack + 2.3.10 + + + + + + + + + ${project.groupId} + jirainstancemanager + ${parent.version} + pom + + + ${project.groupId} + jiraShortcuts + ${parent.version} + pom + + + ${project.groupId} + bitbucketinstancemanager + ${parent.version} + pom + + + + org.codehaus.groovy + groovy-all + ${groovy.version} + provided + pom + + + org.codehaus.groovy + groovy-yaml + ${groovy.version} + + + org.codehaus.groovy + groovy-json + ${groovy.version} + + + + org.codehaus.groovy + groovy-ant + ${groovy.version} + + + org.spockframework + spock-core + ${spock-core.version} + test + + + + + + + + + + + + org.slf4j + slf4j-api + 2.0.1 + + + org.slf4j + slf4j-simple + 2.0.0 + + + + org.apache.commons + commons-compress + 1.21 + + + + commons-io + commons-io + 2.11.0 + + + + + + + + de.gesellix + docker-client + 2023-08-16T08-25-00 + + + + + + + org.junit.jupiter + junit-jupiter-api + 5.9.0 + test + + + org.junit.jupiter + junit-jupiter-engine + 5.9.0 + test + + + + + + + + + + + + + + org.apache.maven.plugins + maven-source-plugin + 3.2.0 + + + attach-sources + generate-sources + + jar-no-fork + test-jar-no-fork + + + + + + org.codehaus.gmavenplus + gmavenplus-plugin + 2.0.0 + + + + addSources + addTestSources + compile + compileTests + + + + execute + + execute + + + + + + org.codehaus.groovy + groovy + ${groovy.version} + runtime + + + + + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.0.0-M7 + + + **/*Spec.class + **/*Test.java + + true + + + + + + + + 11 + 11 + UTF-8 + + + + + \ No newline at end of file diff --git a/src/main/groovy/com/eficode/devstack/container/Container.groovy b/devstack/src/main/groovy/com/eficode/devstack/container/Container.groovy similarity index 89% rename from src/main/groovy/com/eficode/devstack/container/Container.groovy rename to devstack/src/main/groovy/com/eficode/devstack/container/Container.groovy index f684c33..d690736 100644 --- a/src/main/groovy/com/eficode/devstack/container/Container.groovy +++ b/devstack/src/main/groovy/com/eficode/devstack/container/Container.groovy @@ -13,7 +13,6 @@ import de.gesellix.docker.remote.api.ContainerSummary import de.gesellix.docker.remote.api.EndpointSettings import de.gesellix.docker.remote.api.ExecConfig import de.gesellix.docker.remote.api.HostConfig -import de.gesellix.docker.remote.api.IdResponse import de.gesellix.docker.remote.api.Mount import de.gesellix.docker.remote.api.Network import de.gesellix.docker.remote.api.NetworkCreateRequest @@ -75,6 +74,26 @@ trait Container { } } + /** + * Prepare mounting of an existing or new volume + * @param volumeName The name of the volume to create, or an existing one to mount + * @param target Where to mount it in the container + * @param readOnly If it should be read only or not + */ + void prepareVolumeMount(String volumeName, String target, boolean readOnly = true) { + + Mount newMount = new Mount().tap { m -> + m.source = volumeName + m.target = target + m.readOnly = readOnly + m.type = Mount.Type.Volume + } + + if (!self.mounts.find { it.source == volumeName && it.target == target }) { + self.mounts.add(newMount) + } + } + ContainerCreateRequest setupContainerCreateRequest() { @@ -124,7 +143,7 @@ trait Container { ContainerCreateRequest containerCreateRequest = setupContainerCreateRequest() if (cmd.size()) { - containerCreateRequest.cmd = cmd.collect {it.toString()} + containerCreateRequest.cmd = cmd.collect { it.toString() } } if (entrypoint.size()) { @@ -342,16 +361,16 @@ trait Container { } - File createTar(ArrayList filePaths, String outputPath) { + File createTar(ArrayList filePaths, String outputPath, ArrayList ignorePaths = []) { log.info("Creating tar file:" + outputPath) log.debug("\tUsing source paths:") filePaths.each { log.debug("\t\t$it") } - File outputFile = new File(outputPath) TarArchiveOutputStream tarArchive = new TarArchiveOutputStream(Files.newOutputStream(outputFile.toPath())) + tarArchive.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX) log.info("\tProcessing files") filePaths.each { filePath -> @@ -368,28 +387,40 @@ trait Container { String path = ResourceGroovyMethods.relativePath(newEntryFile, subFile) log.trace("\t" * 4 + "Processing sub file:" + path) - TarArchiveEntry entry = new TarArchiveEntry(subFile, path) - entry.setSize(subFile.size()) - tarArchive.putArchiveEntry(entry) - tarArchive.write(subFile.bytes) - tarArchive.closeArchiveEntry() - log.trace("\t" * 5 + "Added to archive") + if (ignorePaths.any { subFile.absolutePath.matches(it) }) { + log.trace("\t" * 5 + "File matches a path that is to be ignored, will not process further") + } else { + TarArchiveEntry entry = new TarArchiveEntry(subFile, path) + entry.setSize(subFile.size()) + tarArchive.putArchiveEntry(entry) + tarArchive.write(subFile.bytes) + tarArchive.closeArchiveEntry() + log.trace("\t" * 5 + "Added to archive") + } + + } } else { log.trace("\t" * 4 + "Processing file:" + newEntryFile.name) - TarArchiveEntry entry = new TarArchiveEntry(newEntryFile, newEntryFile.name) - entry.setSize(newEntryFile.size()) - tarArchive.putArchiveEntry(entry) - tarArchive.write(newEntryFile.bytes) - tarArchive.closeArchiveEntry() - log.trace("\t" * 5 + "Added to archive") + if (ignorePaths.any { newEntryFile.absolutePath.matches(it) }) { + log.trace("\t" * 5 + "File matches a path that is to be ignored, will not process further") + }else { + TarArchiveEntry entry = new TarArchiveEntry(newEntryFile, newEntryFile.name) + entry.setSize(newEntryFile.size()) + tarArchive.putArchiveEntry(entry) + tarArchive.write(newEntryFile.bytes) + tarArchive.closeArchiveEntry() + log.trace("\t" * 5 + "Added to archive") + } } } tarArchive.finish() + log.info("\tFinished creating TAR file:" + outputFile.absolutePath) + log.debug("\t\t" + (outputFile.size() / (1024 * 1024)).round() + "MB") return outputFile @@ -429,6 +460,15 @@ trait Container { } + /** + * Gets the home path for the containers default user + * @return ex: /home/user + */ + String getHomePath() { + runBashCommandInContainer("pwd").find {true} + } + + /** * Creates a network of the type bridge, or returns an existing one if one with the same name exists * @param networkName name of the network @@ -637,12 +677,12 @@ trait Container { boolean replaceFileInContainer(String content, String filePath, boolean verify = false) { ArrayList out = runBashCommandInContainer("cat > $filePath <<- 'EOF'\n" + content + "\nEOF") - assert out.isEmpty() : "Unexpected output when replacing file $filePath: " + out.join("\n") + assert out.isEmpty(): "Unexpected output when replacing file $filePath: " + out.join("\n") if (verify) { - ArrayListrawOut = runBashCommandInContainer("cat " + filePath) + ArrayList rawOut = runBashCommandInContainer("cat " + filePath) String readOut = rawOut.join() - assert readOut.trim() == content.trim() : "Error when verifying that the file $filePath was replaced" + assert readOut.trim() == content.trim(): "Error when verifying that the file $filePath was replaced" return true } @@ -672,10 +712,18 @@ trait Container { } - boolean copyFileToContainer(String srcFilePath, String destinationDirectory) { + /** + * Creates a temporary tar, copies it to the container and extracts it there + * @param srcFilePath Local path to copy, will find directories/files recursively + * @param destinationDirectory The destination path in the container, must already exist and be absolut + * @param ignorePaths If these regex patterns matches the path/name of a file it wont be copied over. + * ex: [".*\\.git.*"] + * @return true on success + */ + boolean copyFileToContainer(String srcFilePath, String destinationDirectory, ArrayList ignorePaths = []) { - File tarFile = createTar([srcFilePath], Files.createTempFile("docker_upload", ".tar").toString()) + File tarFile = createTar([srcFilePath], Files.createTempFile("docker_upload", ".tar").toString(), ignorePaths) dockerClient.putArchive(self.containerId, destinationDirectory, tarFile.newDataInputStream()) return tarFile.delete() @@ -684,6 +732,8 @@ trait Container { static class ContainerCallback implements StreamCallback { + + Logger log = LoggerFactory.getLogger(ContainerCallback.class) ArrayList output = [] @Override @@ -693,6 +743,7 @@ trait Container { } else { output.add(o.toString()) } + log.debug(output.last()) } } @@ -759,7 +810,6 @@ trait Container { } - /** * Creates a temporary container, runs a command, exits and removes container * @param container a container object that hasnt yet been created @@ -774,7 +824,7 @@ trait Container { * @param dockerCertPath * @return An array of the container logs, or just an array containing container id if timeoutMs == 0 */ - static ArrayList runCmdAndRm( ArrayList cmd, long timeoutMs, ArrayList mounts = [], String dockerHost = "", String dockerCertPath = "") { + static ArrayList runCmdAndRm(ArrayList cmd, long timeoutMs, ArrayList mounts = [], String dockerHost = "", String dockerCertPath = "") { Container container = this.getConstructor(String, String).newInstance(dockerHost, dockerCertPath) @@ -782,12 +832,10 @@ trait Container { Logger log = LoggerFactory.getLogger(this.class) - log.info("Creating a $container.class.simpleName and running:") log.info("\tCmd:" + cmd) - try { container.containerName = container.containerName + "-cmd-" + System.currentTimeMillis().toString()[-5..-1] @@ -821,7 +869,7 @@ trait Container { log.info("\tContainer finisehd or timed out after ${System.currentTimeMillis() - start}ms") if (container.running) { - log.info("\t"*2 + "Stopping container forcefully.") + log.info("\t" * 2 + "Stopping container forcefully.") ArrayList containerOut = container.containerLogs assert container.stopAndRemoveContainer(1): "Error stopping and removing CMD container after it timed out" @@ -829,7 +877,6 @@ trait Container { } - ArrayList containerOut = container.containerLogs log.info("\tReturning ${containerOut.size()} log lines") @@ -843,8 +890,8 @@ trait Container { try { container.stopAndRemoveContainer(1) - } catch (ignored){} - + } catch (ignored) { + } throw ex diff --git a/src/main/groovy/com/eficode/devstack/container/impl/AlpineContainer.groovy b/devstack/src/main/groovy/com/eficode/devstack/container/impl/AlpineContainer.groovy similarity index 100% rename from src/main/groovy/com/eficode/devstack/container/impl/AlpineContainer.groovy rename to devstack/src/main/groovy/com/eficode/devstack/container/impl/AlpineContainer.groovy diff --git a/src/main/groovy/com/eficode/devstack/container/impl/BitbucketContainer.groovy b/devstack/src/main/groovy/com/eficode/devstack/container/impl/BitbucketContainer.groovy similarity index 100% rename from src/main/groovy/com/eficode/devstack/container/impl/BitbucketContainer.groovy rename to devstack/src/main/groovy/com/eficode/devstack/container/impl/BitbucketContainer.groovy diff --git a/src/main/groovy/com/eficode/devstack/container/impl/DoodContainer.groovy b/devstack/src/main/groovy/com/eficode/devstack/container/impl/DoodContainer.groovy similarity index 100% rename from src/main/groovy/com/eficode/devstack/container/impl/DoodContainer.groovy rename to devstack/src/main/groovy/com/eficode/devstack/container/impl/DoodContainer.groovy diff --git a/devstack/src/main/groovy/com/eficode/devstack/container/impl/GroovyContainer.groovy b/devstack/src/main/groovy/com/eficode/devstack/container/impl/GroovyContainer.groovy new file mode 100644 index 0000000..4dd887b --- /dev/null +++ b/devstack/src/main/groovy/com/eficode/devstack/container/impl/GroovyContainer.groovy @@ -0,0 +1,319 @@ +package com.eficode.devstack.container.impl +import com.eficode.devstack.container.Container + +import java.util.regex.Matcher +import java.util.regex.Pattern + + +class GroovyContainer implements Container{ + + String containerName = "groovy-container" + String containerMainPort = "" + String containerImage = "groovy" + String containerImageTag = "latest" + + boolean installMavenOnStartup + boolean installGitOnStartup + static final String mvnUrl = "https://dlcdn.apache.org/maven/maven-3/3.9.4/binaries/apache-maven-3.9.4-bin.tar.gz" + static final String mavenNameAndVersion = mvnUrl.substring(mvnUrl.lastIndexOf("/") + 1,mvnUrl.lastIndexOf("-bin.tar.gz")) //ex: apache-maven-3.9.0 + + + GroovyContainer(String dockerHost = "", String dockerCertPath = "") { + if (dockerHost && dockerCertPath) { + assert setupSecureRemoteConnection(dockerHost, dockerCertPath): "Error setting up secure remote docker connection" + } + } + + /** + * Runs a groovy script in the container + * @param scriptText Text of the script that should be run + * @param options Optional options to pass to the groovy process: ex: -Dorg.slf4j.simpleLogger.defaultLogLevel=trace + * @param arguments Optional arguments to pass to the script + * @param timeOutS + * @param containerUser What user to run the script as + * @return An arraylist containing console output from the script + */ + ArrayList runScriptInContainer(String scriptText,String options ="", String arguments = "", long timeOutS = 120, String containerUser = "groovy") { + assert replaceFileInContainer(scriptText, "/home/groovy/userScript.groovy") : "Error uploading script to container $id" + return runBashCommandInContainer("groovy $options /home/groovy/userScript.groovy $arguments 2>&1 | tee /var/log/userScript.log" , timeOutS, containerUser) + + } + + /** + * Creates a container that wont automatically terminate immediately + * @return container id + */ + String createSleepyContainer() { + return createContainer([],["/bin/bash" ,"-c" ,"trap \"exit\" SIGINT SIGTERM && tail -F /var/log/* /var/log/userScript.log"]) + } + + /** + * Creates a container intended for building Groovy maven projects + * @return Container id + */ + String createGroovyBuildContainer() { + + + prepareCustomEnvVar(["PATH=/opt/$mavenNameAndVersion/bin:/opt/java/openjdk/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin".toString()]) + installMavenOnStartup = true + installGitOnStartup = true + return createContainer([],["/bin/bash" ,"-c" ,"trap \"exit\" SIGINT SIGTERM && tail -F /var/log/* /var/log/userScript.log"]) + } + + void useGroovy3() { + this.containerImageTag = "3.0-jdk11-jammy" + } + + void setGroovyVersion(String version) { + + if (version == "latest") { + this.containerImageTag = version + }else { + this.containerImageTag = "$version-jdk11-jammy" + } + + } + + + /** + * Used to mount the local m2 cache, usefull for speeding up maven dependency fetghing + * Will mount container path /home/groovy/.m2/repository + * NOTE will mount readOnly = false + * @param srcPath, defaults to $USERHOME/.m2/repository + */ + void setUseLocalM2Cache(String srcPath = "") { + + String sourcePath = srcPath ?: System.getProperty("user.home") + "/.m2/repository" + prepareBindMount(sourcePath, "/home/groovy/.m2/repository", false) + } + + + boolean installMaven() { + String mvnInstallScript = ""+ + "wget -q $mvnUrl && " + + "tar -xvf ${mvnUrl.substring(mvnUrl.lastIndexOf("/") + 1)} && " + + "mv $mavenNameAndVersion /opt/ && echo Status:\$?" + + ArrayList cmdOut = runBashCommandInContainer(mvnInstallScript, 60000, "root") + assert cmdOut.contains("Status:0") : "Error installing maven" + + + assert runBashCommandInContainer("mvn --version" ).any {it.contains("/opt/$mavenNameAndVersion")} : "Error running maven after install" + assert runBashCommandInContainer("chown -R groovy:groovy /home/groovy && echo Status:\$?", 10000, "root").toString().contains("Status:0") : "Error changing groovy home owner" + + return true + } + + boolean installGit() { + + String installGitScript = "" + + "apt update && " + + "apt install -y git && echo Status:\$?" + + assert runBashCommandInContainer(installGitScript, 30000, "root").toString().contains("Status:0") : "Error install git" + + assert runBashCommandInContainer("git --version && echo Status:\$?").toString().contains("Status:0") : "Error confirming git was installed" + + return true + + } + + boolean isGitInstalled() { + return runBashCommandInContainer("which git && echo Status:\$?").toString().contains("Status:0") + } + + /** + * Checks out a git repo + * @param gitUrl SSH or HTTP url to check out + * @param branch (optional) The branch to check out, if null the default one will be used + * @return The containers filepath to the checked out repo + */ + String cloneGitRepo(String gitUrl, String branch=null) { + + log.info("Cloning Git Repo") + log.info("\tCloning Git Repo:" + gitUrl) + log.info("\tUsing Git Branch:" + (branch ? branch : "(default)")) + + + if (!gitInstalled) { + log.info("\tGit is not installed, installing it now") + assert installGit() : "Error installing git" + log.debug("\t"*2 + "Finished installing git") + } + + String repoName = gitUrl.substring(gitUrl.lastIndexOf("/")+1) + repoName = repoName.endsWith(".git") ? repoName[0..-5] : repoName + + String gitCloneCmd = "rm -rf $repoName && git clone ${branch ? "-b $branch " : ""}$gitUrl $repoName && echo Status:\$?" + + + log.info("\tStarting git clone") + log.debug("\t"*2 + "Using Git Cmd:" + gitCloneCmd) + long start = System.currentTimeSeconds() + ArrayList cloneOutput = runBashCommandInContainer(gitCloneCmd, 120) + assert cloneOutput.toString().contains("Status:0") : "Error cloning Git Repo:" + cloneOutput.join("\n") + log.debug("\t"*2 + "Finished clone after:" + (System.currentTimeSeconds() - start)) + String outputDir = runBashCommandInContainer("cd $repoName && pwd").find {true} + assert outputDir : "Error determining git output dir" + log.info("\t"*2 + "Checked out repo to:" + outputDir) + + return outputDir + + + } + + /** + * Resolve the depencies of a Maven project. + * Usefull to pre-cache them + * @param projectRoot Where is the project root + * @param pomFile Name of the pom file to resolve dependencies for, located in $projectRoot, defaults to pom.xml + * @return true on success + */ + boolean resolveMavenDependencies(String projectRoot, String pomFile = "pom.xml") { + + log.info("Resolving Maven dependencies") + log.info("\tProject root:" + projectRoot) + + log.debug("\t"*2 + "Starting dependency resolve") + long start = System.currentTimeSeconds() + ArrayListresolveOut = runBashCommandInContainer("cd $projectRoot && mvn dependency:resolve -f $pomFile && echo Status:\$?", 1200) + log.debug("\t" *2 + "Finished resolve after " + (System.currentTimeSeconds()- start)) + assert resolveOut.toString().contains("Status:0") + + log.info("\tFinished resolving dependencies") + + return true + + } + + /** + * Generate a new effective pom based on the profiles in a source pom + * @param projectRoot Root of the project where $srcPom is located + * @param mavenProfile The profil/profiles to use, ex: groovy-3,groovy-3.0.14 + * @param dstPom The destination/output pom name + * @param srcPom (Optional) The src pom, defaults to pom.xml + * @return + */ + boolean generateEffectivePom(String projectRoot, String mavenProfile, String dstPom = "effective-pom.xml", String srcPom = "pom.xml" ) { + + log.info("Generating effective pom") + log.info("\tProject root:" + projectRoot) + log.info("\tUsing maven profile:" + mavenProfile) + log.info("\tSource Pom will be::" + srcPom) + log.info("\tDestination Pom will be::" + dstPom) + + ArrayListgenOut = runBashCommandInContainer("cd $projectRoot && mvn help:effective-pom -f $srcPom -P $mavenProfile -Doutput=$dstPom && echo Status:\$?", 10) + assert genOut.toString().contains("Status:0") : "Error generating effective pom:" + genOut.join("\n") + + return true + + } + + + /** + * Run "maven install" for a project + * @param projectRoot Root of project + * @param pom (Optional) The pom file in projectRoot to use, defaults to pom.xml + * @param mavenParams Additional parameters to pass + * @return An array containing the complete file paths of the installed files (in the container) + */ + ArrayList installMavenProject(String projectRoot, String pom = "pom.xml", String mavenParams = "") { + + log.info("Installing a maven project") + log.info("\tProject root:" + projectRoot) + + ArrayListinstallOut = runBashCommandInContainer("cd $projectRoot && mvn install -f $pom $mavenParams && echo Status:\$?", (5*60)) + + if (log.traceEnabled) { + log.trace("\tmvn install output:") + installOut.each {log.trace("\t"*2 + it)} + } + assert installOut.toString().contains("Status:0") : "Error generating effective pom:" + installOut.join("\n") + log.info("\tMaven installed finished") + + log.debug("\tCollecting paths of installed files") + ArrayListinstalledFiles = [] + Pattern installPattern = ~/${projectRoot.replace("/", "\\/")}.*? to (.*)$/ + installOut.findAll {it.contains( "Installing " + projectRoot)}.each {logRow -> + + log.trace("\t"*2+ "Log row contains installed file:" + logRow) + + Matcher matcher = logRow =~installPattern + + ArrayList matches = matcher.findAll {true}.flatten() + if (matches.size() == 2) { + installedFiles.add(matches[1]) + log.trace("\t"*3 + "Extracted file path:" + matches[1]) + }else { + log.warn("Error extracting installed file path from log:" + logRow) + } + + } + log.info("\tDetected ${installedFiles.size()} files installed by maven") + log.info("\tFinished installing maven project") + + return installedFiles + + } + + + void runSpockTest(String jarPath) { + + + + String testDir = "/tmp/" + jarPath.substring(jarPath.lastIndexOf("/") + 1) + "-spock" + + String extractScript = "" + + "rm -rf $testDir && " + + "mkdir -p $testDir && " + + "cd $testDir && " + + "jar xf $jarPath && " + + "echo Status:\$?" + + assert runBashCommandInContainer(extractScript, 60 ).toString().contains("Status:0") : "Error extracting jar $jarPath" + + String resolvePomScript = "" + + "cd $testDir &&" + + "find META-INF/ -name pom.xml" + + ArrayListout = runBashCommandInContainer(resolvePomScript) + assert out.size() == 1 : "Error resolving pom file in $testDir/META-INF/" + String pomRelPath = out.first() + + String buildClasspathScript = "cd $testDir &&" + + "mvn dependency:build-classpath -f $pomRelPath -Dmdep.outputFile=$testDir/classpath.txt && " + + "CLASSPATH=\$(cat classpath.txt) && " + + "echo CLASSPATH-OUT &&" + + "echo \$CLASSPATH" + //"cat classpath.txt" + + //"CLASSPATH=\$(cat classpath.txt) && export CLASSPATH" + + out = runBashCommandInContainer(buildClasspathScript, 120) + assert out.find {it.contains("CLASSPATH-OUT")} + + out = runBashCommandInContainer("echo \$CLASSPATH") + String classPath = out.get(out.findIndexOf {it.contains("CLASSPATH-OUT")} + 1) + + "" + + + } + + @Override + boolean runOnFirstStartup() { + + boolean success = true + if (installMavenOnStartup) { + success = success && installMaven() + } + + if (installGitOnStartup) { + success = success && installGit() + } + + return success + + } + +} diff --git a/src/main/groovy/com/eficode/devstack/container/impl/GroovyDoodContainer.groovy b/devstack/src/main/groovy/com/eficode/devstack/container/impl/GroovyDoodContainer.groovy similarity index 100% rename from src/main/groovy/com/eficode/devstack/container/impl/GroovyDoodContainer.groovy rename to devstack/src/main/groovy/com/eficode/devstack/container/impl/GroovyDoodContainer.groovy diff --git a/src/main/groovy/com/eficode/devstack/container/impl/HarborManagerContainer.groovy b/devstack/src/main/groovy/com/eficode/devstack/container/impl/HarborManagerContainer.groovy similarity index 100% rename from src/main/groovy/com/eficode/devstack/container/impl/HarborManagerContainer.groovy rename to devstack/src/main/groovy/com/eficode/devstack/container/impl/HarborManagerContainer.groovy diff --git a/src/main/groovy/com/eficode/devstack/container/impl/JenkinsContainer.groovy b/devstack/src/main/groovy/com/eficode/devstack/container/impl/JenkinsContainer.groovy similarity index 100% rename from src/main/groovy/com/eficode/devstack/container/impl/JenkinsContainer.groovy rename to devstack/src/main/groovy/com/eficode/devstack/container/impl/JenkinsContainer.groovy diff --git a/src/main/groovy/com/eficode/devstack/container/impl/JsmContainer.groovy b/devstack/src/main/groovy/com/eficode/devstack/container/impl/JsmContainer.groovy similarity index 100% rename from src/main/groovy/com/eficode/devstack/container/impl/JsmContainer.groovy rename to devstack/src/main/groovy/com/eficode/devstack/container/impl/JsmContainer.groovy diff --git a/src/main/groovy/com/eficode/devstack/container/impl/NginxContainer.groovy b/devstack/src/main/groovy/com/eficode/devstack/container/impl/NginxContainer.groovy similarity index 100% rename from src/main/groovy/com/eficode/devstack/container/impl/NginxContainer.groovy rename to devstack/src/main/groovy/com/eficode/devstack/container/impl/NginxContainer.groovy diff --git a/src/main/groovy/com/eficode/devstack/container/impl/UbuntuContainer.groovy b/devstack/src/main/groovy/com/eficode/devstack/container/impl/UbuntuContainer.groovy similarity index 100% rename from src/main/groovy/com/eficode/devstack/container/impl/UbuntuContainer.groovy rename to devstack/src/main/groovy/com/eficode/devstack/container/impl/UbuntuContainer.groovy diff --git a/src/main/groovy/com/eficode/devstack/deployment/Deployment.groovy b/devstack/src/main/groovy/com/eficode/devstack/deployment/Deployment.groovy similarity index 100% rename from src/main/groovy/com/eficode/devstack/deployment/Deployment.groovy rename to devstack/src/main/groovy/com/eficode/devstack/deployment/Deployment.groovy diff --git a/src/main/groovy/com/eficode/devstack/deployment/impl/BitbucketH2Deployment.groovy b/devstack/src/main/groovy/com/eficode/devstack/deployment/impl/BitbucketH2Deployment.groovy similarity index 100% rename from src/main/groovy/com/eficode/devstack/deployment/impl/BitbucketH2Deployment.groovy rename to devstack/src/main/groovy/com/eficode/devstack/deployment/impl/BitbucketH2Deployment.groovy diff --git a/devstack/src/main/groovy/com/eficode/devstack/deployment/impl/GroovyBuilderDeployment.groovy b/devstack/src/main/groovy/com/eficode/devstack/deployment/impl/GroovyBuilderDeployment.groovy new file mode 100644 index 0000000..177ec67 --- /dev/null +++ b/devstack/src/main/groovy/com/eficode/devstack/deployment/impl/GroovyBuilderDeployment.groovy @@ -0,0 +1,165 @@ +package com.eficode.devstack.deployment.impl + +import com.eficode.devstack.container.Container +import com.eficode.devstack.container.impl.GroovyContainer +import com.eficode.devstack.deployment.Deployment +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +class GroovyBuilderDeployment implements Deployment { + + Logger log = LoggerFactory.getLogger(this.class) + String friendlyName = "Groovy Builder Deployment" + ArrayList containers = [] + static String m2VolumeName = "groovy-m2-cache" + String dockerHost + String dockerCertPath + + + GroovyBuilderDeployment(String groovyVersion = "latest", String dockerHost = "", String dockerCertPath = "") { + + + containers = [new GroovyContainer(dockerHost, dockerCertPath)] + this.dockerHost = dockerHost + this.dockerCertPath = dockerCertPath + groovyContainer.setGroovyVersion(groovyVersion) + groovyContainer.setContainerName("groovyBuilder") + } + + GroovyContainer getGroovyContainer() { + return containers.find {it instanceof GroovyContainer} as GroovyContainer + } + + @Override + boolean setupDeployment() { + + groovyContainer.containerDefaultNetworks = [this.deploymentNetworkName] + groovyContainer.prepareVolumeMount(m2VolumeName, "/home/groovy/.m2/", false) + groovyContainer.createGroovyBuildContainer() + groovyContainer.startContainer() + + } + + + static NginxFileServer buildAndHost(String localPath, String hostName = "repo.localhost", String port = "8081", String dockerNetwork = "bridge") { + + GroovyBuilderDeployment groovyBuild = new GroovyBuilderDeployment() + groovyBuild.groovyContainer.containerName = "groovy-buildAndHost" + groovyBuild.setupDeployment() + groovyBuild.buildLocalSources(localPath) + groovyBuild.stopAndRemoveDeployment() + + NginxFileServer fileServer = hostM2Volume(hostName, port, dockerNetwork, groovyBuild.m2VolumeName) + + return fileServer + } + + + /** + * Creates an Nginx based file server + * @param hostName Will be used as container name and hostname + * @param port The port where nginx should listen + * @param dockerNetwork The name of the docker network that the file server should be attached to + * @param volumeName Name of the m2 docker volume to share + */ + static NginxFileServer hostM2Volume(String hostName = "repo.localhost", String port = "8081", String dockerNetwork = "bridge", String volumeName = m2VolumeName) { + + NginxFileServer nginx = new NginxFileServer() + nginx.setPort(port) + nginx.container.containerName = hostName + nginx.deploymentNetworkName = dockerNetwork + nginx.container.prepareVolumeMount(volumeName, "/usr/share/nginx/html", true) + nginx.setupDeployment() + nginx.startDeployment() + + + return nginx + + + } + + + /** + * Builds docker engine local source files in a groovy container + * @param localPath The local paths on the docker engine where sources are found, is expected to have a pom.xml file in the root + * @param mavenProfiles Optional maven profile to build + * @param ignorePaths Paths that should be ignored when copying files to the groovy container + * @return A list of built files in the docker container + */ + ArrayList buildLocalSources(String localPath, String mavenProfiles = "", ArrayList ignorePaths = [".*\\.git.*", ".*target/.*", ".*\\.terraform/.*"]) { + + String projectRoot = groovyContainer.getHomePath() + "/projectRoot/" + + log.info("Building JARs from local sources") + + + groovyContainer.runBashCommandInContainer("rm -rf $projectRoot && mkdir -p $projectRoot") + assert groovyContainer.copyFileToContainer(localPath, projectRoot, ignorePaths) + groovyContainer.runBashCommandInContainer("chown groovy:groovy -R $projectRoot", 10, "root") + + + log.info("\tFinished checking copying sources from local path:" + localPath) + + String pomFileName = "pom.xml" + + //If maven profiles are supplied, we generate a new effective pom file + if (mavenProfiles) { + assert groovyContainer.generateEffectivePom(projectRoot, mavenProfiles, "effective-pom.xml") : "Error generating effective pom" + pomFileName = "effective-pom.xml" + log.debug("\tGenerated a new pom file based on Maven Profile(s):" + mavenProfiles ) + } + + log.info("\tResolving dependencies for pom: " + projectRoot + "/"+pomFileName) + groovyContainer.resolveMavenDependencies(projectRoot,pomFileName) + log.info("\t"* 2 + "Finished resolving dependencies") + + + ArrayList installedFiles = groovyContainer.installMavenProject(projectRoot, pomFileName) + + log.info("\tBuilt ${installedFiles.size()} files") + if (log.isDebugEnabled()) { + installedFiles.each { + log.debug("\t\t" + it) + } + } + + return installedFiles + } + + + + + + void buildGroovyJarsFromGit(String gitUrl, String gitBranch = "", String mavenProfiles = "") { + + log.info("Building JARs from Git Repo") + String projectRoot = groovyContainer.cloneGitRepo(gitUrl, gitBranch) + log.info("\tFinished checking out git repo to container path:" + projectRoot) + + String pomFileName = "pom.xml" + + //If maven profiles are supplied, we generate a new effective pom file + if (mavenProfiles) { + assert groovyContainer.generateEffectivePom(projectRoot, mavenProfiles, "effective-pom.xml") : "Error generating effective pom" + pomFileName = "effective-pom.xml" + log.debug("\tGenerated a new pom file based on Maven Profile(s):" + mavenProfiles ) + } + + log.info("\tResolving dependencies for pom: " + projectRoot + "/"+pomFileName) + groovyContainer.resolveMavenDependencies(projectRoot,pomFileName) + log.info("\t"* 2 + "Finished resolving dependencies") + + + ArrayList installedFiles = groovyContainer.installMavenProject(projectRoot, pomFileName) + + + + + "" + + + + } + + +} diff --git a/src/main/groovy/com/eficode/devstack/deployment/impl/HarborDeployment.groovy b/devstack/src/main/groovy/com/eficode/devstack/deployment/impl/HarborDeployment.groovy similarity index 100% rename from src/main/groovy/com/eficode/devstack/deployment/impl/HarborDeployment.groovy rename to devstack/src/main/groovy/com/eficode/devstack/deployment/impl/HarborDeployment.groovy diff --git a/src/main/groovy/com/eficode/devstack/deployment/impl/JenkinsAndHarborDeployment.groovy b/devstack/src/main/groovy/com/eficode/devstack/deployment/impl/JenkinsAndHarborDeployment.groovy similarity index 100% rename from src/main/groovy/com/eficode/devstack/deployment/impl/JenkinsAndHarborDeployment.groovy rename to devstack/src/main/groovy/com/eficode/devstack/deployment/impl/JenkinsAndHarborDeployment.groovy diff --git a/src/main/groovy/com/eficode/devstack/deployment/impl/JenkinsDeployment.groovy b/devstack/src/main/groovy/com/eficode/devstack/deployment/impl/JenkinsDeployment.groovy similarity index 100% rename from src/main/groovy/com/eficode/devstack/deployment/impl/JenkinsDeployment.groovy rename to devstack/src/main/groovy/com/eficode/devstack/deployment/impl/JenkinsDeployment.groovy diff --git a/src/main/groovy/com/eficode/devstack/deployment/impl/JsmAndBitbucketH2Deployment.groovy b/devstack/src/main/groovy/com/eficode/devstack/deployment/impl/JsmAndBitbucketH2Deployment.groovy similarity index 100% rename from src/main/groovy/com/eficode/devstack/deployment/impl/JsmAndBitbucketH2Deployment.groovy rename to devstack/src/main/groovy/com/eficode/devstack/deployment/impl/JsmAndBitbucketH2Deployment.groovy diff --git a/src/main/groovy/com/eficode/devstack/deployment/impl/JsmH2Deployment.groovy b/devstack/src/main/groovy/com/eficode/devstack/deployment/impl/JsmH2Deployment.groovy similarity index 100% rename from src/main/groovy/com/eficode/devstack/deployment/impl/JsmH2Deployment.groovy rename to devstack/src/main/groovy/com/eficode/devstack/deployment/impl/JsmH2Deployment.groovy diff --git a/src/main/groovy/com/eficode/devstack/deployment/impl/NginxFileServer.groovy b/devstack/src/main/groovy/com/eficode/devstack/deployment/impl/NginxFileServer.groovy similarity index 97% rename from src/main/groovy/com/eficode/devstack/deployment/impl/NginxFileServer.groovy rename to devstack/src/main/groovy/com/eficode/devstack/deployment/impl/NginxFileServer.groovy index a02cb2a..c98cb68 100644 --- a/src/main/groovy/com/eficode/devstack/deployment/impl/NginxFileServer.groovy +++ b/devstack/src/main/groovy/com/eficode/devstack/deployment/impl/NginxFileServer.groovy @@ -79,6 +79,7 @@ class NginxFileServer implements Deployment { @Override boolean setupDeployment() { + container.containerDefaultNetworks = [this.deploymentNetworkName] return containers.first().createContainer() != null } } diff --git a/src/main/groovy/com/eficode/devstack/simplelogger.properties b/devstack/src/main/groovy/com/eficode/devstack/simplelogger.properties similarity index 100% rename from src/main/groovy/com/eficode/devstack/simplelogger.properties rename to devstack/src/main/groovy/com/eficode/devstack/simplelogger.properties diff --git a/src/main/groovy/com/eficode/devstack/util/ImageBuilder.groovy b/devstack/src/main/groovy/com/eficode/devstack/util/ImageBuilder.groovy similarity index 100% rename from src/main/groovy/com/eficode/devstack/util/ImageBuilder.groovy rename to devstack/src/main/groovy/com/eficode/devstack/util/ImageBuilder.groovy diff --git a/src/main/groovy/com/eficode/devstack/util/TimeMachine.groovy b/devstack/src/main/groovy/com/eficode/devstack/util/TimeMachine.groovy similarity index 100% rename from src/main/groovy/com/eficode/devstack/util/TimeMachine.groovy rename to devstack/src/main/groovy/com/eficode/devstack/util/TimeMachine.groovy diff --git a/src/main/resources/com/eficode/devstack/deployment/jira/scripts/CreateBitbucketLink.groovy b/devstack/src/main/resources/com/eficode/devstack/deployment/jira/scripts/CreateBitbucketLink.groovy similarity index 93% rename from src/main/resources/com/eficode/devstack/deployment/jira/scripts/CreateBitbucketLink.groovy rename to devstack/src/main/resources/com/eficode/devstack/deployment/jira/scripts/CreateBitbucketLink.groovy index 96d0870..eb602ba 100644 --- a/src/main/resources/com/eficode/devstack/deployment/jira/scripts/CreateBitbucketLink.groovy +++ b/devstack/src/main/resources/com/eficode/devstack/deployment/jira/scripts/CreateBitbucketLink.groovy @@ -1,4 +1,4 @@ -package com.eficode.atlassian.JiraShortcuts.test.resources.jiraLocalScripts +package com.eficode.devstack.deployment.jira.scripts /** * A script executed in JIRA by Scriptrunner diff --git a/src/main/simplelogger.properties b/devstack/src/main/simplelogger.properties similarity index 100% rename from src/main/simplelogger.properties rename to devstack/src/main/simplelogger.properties diff --git a/src/test/groovy/com/eficode/devstack/DevStackSpec.groovy b/devstack/src/test/groovy/com/eficode/devstack/DevStackSpec.groovy similarity index 97% rename from src/test/groovy/com/eficode/devstack/DevStackSpec.groovy rename to devstack/src/test/groovy/com/eficode/devstack/DevStackSpec.groovy index fd17514..33ba972 100644 --- a/src/test/groovy/com/eficode/devstack/DevStackSpec.groovy +++ b/devstack/src/test/groovy/com/eficode/devstack/DevStackSpec.groovy @@ -19,9 +19,11 @@ import java.util.regex.Matcher class DevStackSpec extends Specification { @Shared - String dockerRemoteHost = "https://docker.domain.se:2376" + //String dockerRemoteHost = "https://docker.domain.se:2376" + String dockerRemoteHost = "" @Shared - String dockerCertPath = "~/.docker/" + //String dockerCertPath = "~/.docker/" + String dockerCertPath = "" @Shared DockerClientDS dockerClient diff --git a/src/test/groovy/com/eficode/devstack/container/ContainerTest.groovy b/devstack/src/test/groovy/com/eficode/devstack/container/ContainerTest.groovy similarity index 100% rename from src/test/groovy/com/eficode/devstack/container/ContainerTest.groovy rename to devstack/src/test/groovy/com/eficode/devstack/container/ContainerTest.groovy diff --git a/src/test/groovy/com/eficode/devstack/container/impl/BitbucketContainerTest.groovy b/devstack/src/test/groovy/com/eficode/devstack/container/impl/BitbucketContainerTest.groovy similarity index 100% rename from src/test/groovy/com/eficode/devstack/container/impl/BitbucketContainerTest.groovy rename to devstack/src/test/groovy/com/eficode/devstack/container/impl/BitbucketContainerTest.groovy diff --git a/src/test/groovy/com/eficode/devstack/container/impl/DoodContainerTest.groovy b/devstack/src/test/groovy/com/eficode/devstack/container/impl/DoodContainerTest.groovy similarity index 95% rename from src/test/groovy/com/eficode/devstack/container/impl/DoodContainerTest.groovy rename to devstack/src/test/groovy/com/eficode/devstack/container/impl/DoodContainerTest.groovy index 0171d9a..7697144 100644 --- a/src/test/groovy/com/eficode/devstack/container/impl/DoodContainerTest.groovy +++ b/devstack/src/test/groovy/com/eficode/devstack/container/impl/DoodContainerTest.groovy @@ -12,7 +12,7 @@ class DoodContainerTest extends DevStackSpec { dockerCertPath = "~/.docker/" - log = LoggerFactory.getLogger(DoodContainerTest.class) + DevStackSpec.log = LoggerFactory.getLogger(DoodContainerTest.class) cleanupContainerNames = ["dood.domain.se"] diff --git a/devstack/src/test/groovy/com/eficode/devstack/container/impl/GroovyContainerTest.groovy b/devstack/src/test/groovy/com/eficode/devstack/container/impl/GroovyContainerTest.groovy new file mode 100644 index 0000000..c92c518 --- /dev/null +++ b/devstack/src/test/groovy/com/eficode/devstack/container/impl/GroovyContainerTest.groovy @@ -0,0 +1,57 @@ +package com.eficode.devstack.container.impl + +import com.eficode.devstack.DevStackSpec +import com.eficode.devstack.deployment.impl.GroovyBuilderDeployment +import com.eficode.devstack.deployment.impl.NginxFileServer +import kong.unirest.Unirest +import org.slf4j.LoggerFactory + +class GroovyContainerTest extends DevStackSpec{ + + def setupSpec() { + + + DevStackSpec.log = LoggerFactory.getLogger(DoodContainerTest.class) + + cleanupContainerNames = ["groovyBuilder"] + cleanupContainerPorts = [] + + disableCleanup = true + + } + + + String getProjectRootPath() { + + String root = System.getenv("projectRoot") + assert root != "" : "Error determining project root, did you run using the Run Config?" + return root + + } + + + def "Test build DevStack"() { + + + setup: + //GroovyBuilderDeployment groovyBuild = new GroovyBuilderDeployment("3.0.14") + //NginxContainer nginxContainer = new NginxContainer() + //nginxContainer.prepareVolumeMount(groovyBuild.m2VolumeName, "/usr/share/nginx/html", true) + NginxFileServer fileServer = GroovyBuilderDeployment.buildAndHost(projectRootPath) + //groovyBuild.groovyContainer.setUseLocalM2Cache() + + + + expect: + Unirest.get(fileServer.container.containerName + ":" + fileServer.container.containerMainPort) + //groovyBuild.groovyContainer.isCreated() ?: groovyBuild.setupDeployment() + //groovyBuild.buildLocalSources(projectRootPath) + //nginxContainer.createContainer() + //groovyBuild.hostM2Volume("repo.localhost", "8081", "jsm123") + + + + } + + +} diff --git a/src/test/groovy/com/eficode/devstack/container/impl/JenkinsContainerTest.groovy b/devstack/src/test/groovy/com/eficode/devstack/container/impl/JenkinsContainerTest.groovy similarity index 82% rename from src/test/groovy/com/eficode/devstack/container/impl/JenkinsContainerTest.groovy rename to devstack/src/test/groovy/com/eficode/devstack/container/impl/JenkinsContainerTest.groovy index 95c8231..0a3161a 100644 --- a/src/test/groovy/com/eficode/devstack/container/impl/JenkinsContainerTest.groovy +++ b/devstack/src/test/groovy/com/eficode/devstack/container/impl/JenkinsContainerTest.groovy @@ -4,7 +4,6 @@ import com.eficode.devstack.DevStackSpec import de.gesellix.docker.remote.api.ContainerState import de.gesellix.docker.remote.api.core.Frame import de.gesellix.docker.remote.api.core.StreamCallback -import kong.unirest.HttpResponse import kong.unirest.Unirest import kong.unirest.UnirestInstance import org.slf4j.LoggerFactory @@ -18,7 +17,7 @@ class JenkinsContainerTest extends DevStackSpec{ dockerRemoteHost = "https://docker.domain.se:2376" dockerCertPath = "~/.docker/" - log = LoggerFactory.getLogger(JenkinsContainerTest.class) + DevStackSpec.log = LoggerFactory.getLogger(JenkinsContainerTest.class) cleanupContainerNames = ["jenkins.domain.se", "jenkins-agent.domain.se", "localhost"] @@ -45,7 +44,7 @@ class JenkinsContainerTest extends DevStackSpec{ long start = System.currentTimeMillis() //String baseUrl = "http://" + jc.containerName + ":" + jc.containerMainPort + "/login" - log.info("Waiting for Jenkins WEB-UI to become responsive") + DevStackSpec.log.info("Waiting for Jenkins WEB-UI to become responsive") @@ -64,19 +63,19 @@ class JenkinsContainerTest extends DevStackSpec{ try { int status = unirestInstance.get(baseUrl + "/login").socketTimeout(5000).connectTimeout(10000).asEmpty()?.status if (status == 200){ - log.info("\tJenkins is ready and responded with HTTP status:" + status + " after " + ((System.currentTimeMillis() - start)/1000).round() + "s") + DevStackSpec.log.info("\tJenkins is ready and responded with HTTP status:" + status + " after " + ((System.currentTimeMillis() - start)/1000).round() + "s") break } else { - log.info("\tJenkins responded with HTTP status:" + status) + DevStackSpec.log.info("\tJenkins responded with HTTP status:" + status) sleep(2000) } }catch(ex) { - log.warn("\tError accessing Jenkins WEB-UI:" + ex.message) + DevStackSpec.log.warn("\tError accessing Jenkins WEB-UI:" + ex.message) sleep(2000) } }else { - log.error("\tTimed our waiting for Jenkins after:" + ((System.currentTimeMillis() - start)/1000).round() + "s") + DevStackSpec.log.error("\tTimed our waiting for Jenkins after:" + ((System.currentTimeMillis() - start)/1000).round() + "s") throw new TimeoutException("Error waiting for Jenkins WEB to become available:" + baseUrl) } diff --git a/src/test/groovy/com/eficode/devstack/container/impl/JsmContainerTest.groovy b/devstack/src/test/groovy/com/eficode/devstack/container/impl/JsmContainerTest.groovy similarity index 78% rename from src/test/groovy/com/eficode/devstack/container/impl/JsmContainerTest.groovy rename to devstack/src/test/groovy/com/eficode/devstack/container/impl/JsmContainerTest.groovy index c5396a7..7cf270b 100644 --- a/src/test/groovy/com/eficode/devstack/container/impl/JsmContainerTest.groovy +++ b/devstack/src/test/groovy/com/eficode/devstack/container/impl/JsmContainerTest.groovy @@ -19,7 +19,7 @@ class JsmContainerTest extends DevStackSpec { dockerCertPath = "~/.docker/" - log = LoggerFactory.getLogger(JsmContainerTest.class) + DevStackSpec.log = LoggerFactory.getLogger(JsmContainerTest.class) cleanupContainerNames = ["jira.domain.se", "JSM", "Spoc-JSM"] cleanupContainerPorts = [8080] @@ -30,28 +30,28 @@ class JsmContainerTest extends DevStackSpec { def "test isCreated"(String dockerHost, String certPath) { when: - log.info("Testing isCreated") + DevStackSpec.log.info("Testing isCreated") JsmContainer jsm = new JsmContainer(dockerHost, certPath) then: !jsm.isCreated() - log.info("\tDid not return a false positive") + DevStackSpec.log.info("\tDid not return a false positive") when: String containerId = jsm.createContainer() - log.info("\tCreated container:" + containerId) + DevStackSpec.log.info("\tCreated container:" + containerId) then: jsm.isCreated() - log.info("\tisCreated now returns true") + DevStackSpec.log.info("\tisCreated now returns true") when: jsm.stopAndRemoveContainer() ?: {throw new Exception("Error revoming container $containerId")} - log.info("\tRemoved container") + DevStackSpec.log.info("\tRemoved container") then: !jsm.isCreated() - log.info("\tisCreated now again returns false") + DevStackSpec.log.info("\tisCreated now again returns false") where: dockerHost | certPath @@ -80,7 +80,7 @@ class JsmContainerTest extends DevStackSpec { def "test setupContainer"(String dockerHost, String certPath) { setup: - log.info("Testing setup of JSM container using trait method") + DevStackSpec.log.info("Testing setup of JSM container using trait method") JsmContainer jsm = new JsmContainer(dockerHost, certPath) when: @@ -94,7 +94,7 @@ class JsmContainerTest extends DevStackSpec { assert containerInspect.state.running == false : "JSM Container was started even though it should only have been created" assert dockerClient.inspectImage(containerInspect.image).content.repoTags.find {it == "atlassian/jira-servicemanagement:latest"} : "JSM container was created with incorrect Docker image" assert containerInspect.hostConfig.portBindings.containsKey("8080/tcp") : "JSM Container port binding was not setup correctly" - log.info("\tJSM Container was setup correctly") + DevStackSpec.log.info("\tJSM Container was setup correctly") where: @@ -108,7 +108,7 @@ class JsmContainerTest extends DevStackSpec { def "test non standard parameters"(String dockerHost, String certPath) { setup: - log.info("Testing setup of JSM container using dedicated JSM method") + DevStackSpec.log.info("Testing setup of JSM container using dedicated JSM method") JsmContainer jsm = new JsmContainer(dockerHost, certPath) jsm.containerName = "Spoc-JSM" jsm.containerImageTag = "4-ubuntu-jdk11" @@ -125,7 +125,7 @@ class JsmContainerTest extends DevStackSpec { assert containerInspect.state.running == false : "JSM Container was started even though it should only have been created" assert containerInspect.hostConfig.portBindings.containsKey("666/tcp") : "JSM Container port binding was not setup correctly" assert dockerClient.inspectImage(containerInspect.image).content.repoTags.find {it == "atlassian/jira-servicemanagement:4-ubuntu-jdk11"} : "JSM container was created with incorrect Docker image" - log.info("\tJSM Container was setup correctly") + DevStackSpec.log.info("\tJSM Container was setup correctly") where: @@ -141,11 +141,11 @@ class JsmContainerTest extends DevStackSpec { setup: - log.info("Testing stop and removal of JSM container") + DevStackSpec.log.info("Testing stop and removal of JSM container") JsmContainer jsm = new JsmContainer(dockerHost, certPath) when: "Setting up the container with the trait method" - log.info("\tSetting up JSM container using trait method") + DevStackSpec.log.info("\tSetting up JSM container using trait method") String containerId = jsm.createContainer() then: "Removing it should return true" @@ -161,7 +161,7 @@ class JsmContainerTest extends DevStackSpec { when: "Setting up the container with the trait method" - log.info("\tSetting up JSM container using dedicated JSM method") + DevStackSpec.log.info("\tSetting up JSM container using dedicated JSM method") String containerId2 = jsm.createContainer() then: "Removing it should return true" @@ -188,42 +188,42 @@ class JsmContainerTest extends DevStackSpec { String containerSrcPath = "/opt/atlassian/jira/atlassian-jira/WEB-INF/classes/com/atlassian/jira/" String containerDstDir = "/var/atlassian/application-data/jira/" - log.info("Testing copying files to and from JSM container") + DevStackSpec.log.info("Testing copying files to and from JSM container") JsmContainer jsm = new JsmContainer(dockerHost, certPath) String containerId = jsm.createContainer() - log.info("\tCreated container:" + containerId) + DevStackSpec.log.info("\tCreated container:" + containerId) Path tempDir = Files.createTempDirectory("testing-${this.class.simpleName}") - log.info("\tCreated temp dir:" + containerId.toString()) + DevStackSpec.log.info("\tCreated temp dir:" + containerId.toString()) when: "Copying files from container path:" - log.info("\tCopying files from container path:" + containerSrcPath) + DevStackSpec.log.info("\tCopying files from container path:" + containerSrcPath) ArrayListcopiedFiles = jsm.copyFilesFromContainer(containerSrcPath, tempDir.toString() + "/") - log.info("\tCopied ${copiedFiles.size()} files from container") + DevStackSpec.log.info("\tCopied ${copiedFiles.size()} files from container") then: "Several files and directories should have been copied" assert copiedFiles.size() : "No files where copied from container" assert copiedFiles.any {it.directory} : "No directories where copied from container" - log.info("\tCopying files from container appears successful") + DevStackSpec.log.info("\tCopying files from container appears successful") when: "Copying a file to container" assert jsm.startContainer() : "Error starting container" File largestFile = copiedFiles.sort {it.size()}.last() String fileHash = largestFile.bytes.sha256() - log.info("\tCopying file ($largestFile.name) to container path:" + containerDstDir + largestFile.name) - log.debug("\t\tFile size:" + (largestFile.size() * 0.000001).round(1) + "MB") - log.debug("\t\tFile hash:" + fileHash) + DevStackSpec.log.info("\tCopying file ($largestFile.name) to container path:" + containerDstDir + largestFile.name) + DevStackSpec.log.debug("\t\tFile size:" + (largestFile.size() * 0.000001).round(1) + "MB") + DevStackSpec.log.debug("\t\tFile hash:" + fileHash) then: "File should copy without error" jsm.copyFileToContainer(largestFile.path, containerDstDir) - log.info("\tFinished copying file to container") + DevStackSpec.log.info("\tFinished copying file to container") when: "Running a hash in the container" - log.info("Executing hash calculation of file in container") + DevStackSpec.log.info("Executing hash calculation of file in container") ArrayList hashOutput = jsm.runBashCommandInContainer("sha256sum " + containerDstDir + largestFile.name) - log.debug("\tContainer hash output:" + hashOutput) + DevStackSpec.log.debug("\tContainer hash output:" + hashOutput) then: "The container hash and local hash should be identical" assert hashOutput.size() == 1 : "Expected one output row from remote bash command" @@ -234,7 +234,7 @@ class JsmContainerTest extends DevStackSpec { cleanup: - log.info("\tDeleting temp dir:" + tempDir.toString()) + DevStackSpec.log.info("\tDeleting temp dir:" + tempDir.toString()) FileUtils.deleteDirectory(tempDir.toFile()) diff --git a/src/test/groovy/com/eficode/devstack/container/impl/NginxContainerTest.groovy b/devstack/src/test/groovy/com/eficode/devstack/container/impl/NginxContainerTest.groovy similarity index 92% rename from src/test/groovy/com/eficode/devstack/container/impl/NginxContainerTest.groovy rename to devstack/src/test/groovy/com/eficode/devstack/container/impl/NginxContainerTest.groovy index d751ea6..dfa4de3 100644 --- a/src/test/groovy/com/eficode/devstack/container/impl/NginxContainerTest.groovy +++ b/devstack/src/test/groovy/com/eficode/devstack/container/impl/NginxContainerTest.groovy @@ -16,7 +16,7 @@ class NginxContainerTest extends DevStackSpec { dockerCertPath = "~/.docker/" - log = LoggerFactory.getLogger(this.class) + DevStackSpec.log = LoggerFactory.getLogger(this.class) cleanupContainerNames = ["Nginx", "Nginx-File-Server"] cleanupContainerPorts = [80] @@ -108,8 +108,8 @@ class NginxContainerTest extends DevStackSpec { setup: String localNginxRoot = "/tmp/" - log.info("Testing setting nginx root dir") - log.info("\tWill use dir:" + localNginxRoot) + DevStackSpec.log.info("Testing setting nginx root dir") + DevStackSpec.log.info("\tWill use dir:" + localNginxRoot) NginxContainer nginxC = new NginxContainer(dockerHost, certPath) //stopAndRemoveContainer([nginxC.containerName]) @@ -122,14 +122,14 @@ class NginxContainerTest extends DevStackSpec { when: "After creating the container, the inspect result should confirm the mount" String containerId = nginxC.createContainer() - log.info("\tCreated container:" + containerId) + DevStackSpec.log.info("\tCreated container:" + containerId) assert nginxC.startContainer(): "Error starting container" ContainerInspectResponse inspectResponse = dockerClient.inspectContainer(nginxC.id).getContent() - log.info("\tContainer created") + DevStackSpec.log.info("\tContainer created") then: inspectResponse.hostConfig.mounts.find { it.source == localNginxRoot } - log.info("\tDocker API confirms mount was created") + DevStackSpec.log.info("\tDocker API confirms mount was created") when: "Creating a file in the mounted dir" ArrayList cmdOutput = nginxC.runBashCommandInContainer("mkdir -p /usr/share/nginx/html/nginxTest && touch /usr/share/nginx/html/nginxTest/a.file && echo status: \$?") diff --git a/src/test/groovy/com/eficode/devstack/deployment/impl/BitbucketH2DeploymentTest.groovy b/devstack/src/test/groovy/com/eficode/devstack/deployment/impl/BitbucketH2DeploymentTest.groovy similarity index 86% rename from src/test/groovy/com/eficode/devstack/deployment/impl/BitbucketH2DeploymentTest.groovy rename to devstack/src/test/groovy/com/eficode/devstack/deployment/impl/BitbucketH2DeploymentTest.groovy index 9f2dfe9..a0b84b8 100644 --- a/src/test/groovy/com/eficode/devstack/deployment/impl/BitbucketH2DeploymentTest.groovy +++ b/devstack/src/test/groovy/com/eficode/devstack/deployment/impl/BitbucketH2DeploymentTest.groovy @@ -1,11 +1,7 @@ package com.eficode.devstack.deployment.impl import com.eficode.devstack.DevStackSpec -import com.eficode.devstack.util.DockerClientDS -import de.gesellix.docker.engine.DockerClientConfig -import de.gesellix.docker.engine.DockerEnv import kong.unirest.Unirest -import org.apache.commons.io.FileUtils import org.slf4j.LoggerFactory import spock.lang.Shared @@ -23,8 +19,7 @@ class BitbucketH2DeploymentTest extends DevStackSpec{ //dockerCertPath = "~/.docker/" - - log = LoggerFactory.getLogger(BitbucketH2DeploymentTest.class) + DevStackSpec.log = LoggerFactory.getLogger(BitbucketH2DeploymentTest.class) cleanupContainerNames = ["bitbucket.domain.se", "bitbucket2.domain.se" , "localhost"] cleanupContainerPorts = [7990, 7992, 80] diff --git a/src/test/groovy/com/eficode/devstack/deployment/impl/HarborDeploymentTest.groovy b/devstack/src/test/groovy/com/eficode/devstack/deployment/impl/HarborDeploymentTest.groovy similarity index 96% rename from src/test/groovy/com/eficode/devstack/deployment/impl/HarborDeploymentTest.groovy rename to devstack/src/test/groovy/com/eficode/devstack/deployment/impl/HarborDeploymentTest.groovy index 6260da6..827c067 100644 --- a/src/test/groovy/com/eficode/devstack/deployment/impl/HarborDeploymentTest.groovy +++ b/devstack/src/test/groovy/com/eficode/devstack/deployment/impl/HarborDeploymentTest.groovy @@ -14,7 +14,7 @@ class HarborDeploymentTest extends DevStackSpec { //dockerCertPath = "~/.docker/" - log = LoggerFactory.getLogger(HarborDeploymentTest.class) + DevStackSpec.log = LoggerFactory.getLogger(HarborDeploymentTest.class) cleanupContainerPorts = [80] @@ -76,7 +76,7 @@ class HarborDeploymentTest extends DevStackSpec { containerIdsBeforeStart.sort() == hd.getHarborContainers().id.sort() hd.getContainers().every { it.status() == ContainerState.Status.Running } Unirest.get(harborBaseUrl).basicAuth("admin", "Harbor12345").asEmpty().status == 200 - log.info("\tSuccessful harbor Web login! ") + DevStackSpec.log.info("\tSuccessful harbor Web login! ") when: hd.stopAndRemoveDeployment() diff --git a/src/test/groovy/com/eficode/devstack/deployment/impl/JenkinsAndHarborDeploymentTest.groovy b/devstack/src/test/groovy/com/eficode/devstack/deployment/impl/JenkinsAndHarborDeploymentTest.groovy similarity index 96% rename from src/test/groovy/com/eficode/devstack/deployment/impl/JenkinsAndHarborDeploymentTest.groovy rename to devstack/src/test/groovy/com/eficode/devstack/deployment/impl/JenkinsAndHarborDeploymentTest.groovy index 8b29674..12a9998 100644 --- a/src/test/groovy/com/eficode/devstack/deployment/impl/JenkinsAndHarborDeploymentTest.groovy +++ b/devstack/src/test/groovy/com/eficode/devstack/deployment/impl/JenkinsAndHarborDeploymentTest.groovy @@ -12,7 +12,7 @@ class JenkinsAndHarborDeploymentTest extends DevStackSpec { dockerCertPath = "~/.docker/" - log = LoggerFactory.getLogger(JsmH2DeploymentTest.class) + DevStackSpec.log = LoggerFactory.getLogger(JsmH2DeploymentTest.class) cleanupContainerNames = [ diff --git a/src/test/groovy/com/eficode/devstack/deployment/impl/JsmAndBitbucketH2DeploymentTest.groovy b/devstack/src/test/groovy/com/eficode/devstack/deployment/impl/JsmAndBitbucketH2DeploymentTest.groovy similarity index 96% rename from src/test/groovy/com/eficode/devstack/deployment/impl/JsmAndBitbucketH2DeploymentTest.groovy rename to devstack/src/test/groovy/com/eficode/devstack/deployment/impl/JsmAndBitbucketH2DeploymentTest.groovy index 42a578e..dd16039 100644 --- a/src/test/groovy/com/eficode/devstack/deployment/impl/JsmAndBitbucketH2DeploymentTest.groovy +++ b/devstack/src/test/groovy/com/eficode/devstack/deployment/impl/JsmAndBitbucketH2DeploymentTest.groovy @@ -31,7 +31,7 @@ class JsmAndBitbucketH2DeploymentTest extends DevStackSpec { dockerCertPath = "~/.docker/" - log = LoggerFactory.getLogger(JsmH2DeploymentTest.class) + DevStackSpec.log = LoggerFactory.getLogger(JsmH2DeploymentTest.class) cleanupContainerNames = ["jira.domain.se", "jira2.domain.se", "bitbucket.domain.se", "bitbucket2.domain.se"] diff --git a/src/test/groovy/com/eficode/devstack/deployment/impl/JsmH2DeploymentTest.groovy b/devstack/src/test/groovy/com/eficode/devstack/deployment/impl/JsmH2DeploymentTest.groovy similarity index 96% rename from src/test/groovy/com/eficode/devstack/deployment/impl/JsmH2DeploymentTest.groovy rename to devstack/src/test/groovy/com/eficode/devstack/deployment/impl/JsmH2DeploymentTest.groovy index a72655e..833bf26 100644 --- a/src/test/groovy/com/eficode/devstack/deployment/impl/JsmH2DeploymentTest.groovy +++ b/devstack/src/test/groovy/com/eficode/devstack/deployment/impl/JsmH2DeploymentTest.groovy @@ -19,7 +19,7 @@ class JsmH2DeploymentTest extends DevStackSpec { //dockerCertPath = "~/.docker/" - log = LoggerFactory.getLogger(JsmH2DeploymentTest.class) + DevStackSpec.log = LoggerFactory.getLogger(JsmH2DeploymentTest.class) cleanupContainerNames = ["jira.domain.se", "jira2.domain.se", "localhost"] diff --git a/src/test/groovy/com/eficode/devstack/examples/ExamplesTest.groovy b/devstack/src/test/groovy/com/eficode/devstack/examples/ExamplesTest.groovy similarity index 98% rename from src/test/groovy/com/eficode/devstack/examples/ExamplesTest.groovy rename to devstack/src/test/groovy/com/eficode/devstack/examples/ExamplesTest.groovy index 50e45a0..643c652 100644 --- a/src/test/groovy/com/eficode/devstack/examples/ExamplesTest.groovy +++ b/devstack/src/test/groovy/com/eficode/devstack/examples/ExamplesTest.groovy @@ -27,7 +27,7 @@ class ExamplesTest extends DevStackSpec{ dockerRemoteHost = "" - log = LoggerFactory.getLogger(ExamplesTest.class) + DevStackSpec.log = LoggerFactory.getLogger(ExamplesTest.class) cleanupContainerNames = ["groovy-container", "jira.local", "bitbucket.local"] cleanupContainerPorts = [8080,7990] diff --git a/src/test/groovy/simplelogger.properties b/devstack/src/test/groovy/simplelogger.properties similarity index 100% rename from src/test/groovy/simplelogger.properties rename to devstack/src/test/groovy/simplelogger.properties diff --git a/pom.xml b/pom.xml index d618385..ef9a852 100644 --- a/pom.xml +++ b/pom.xml @@ -5,225 +5,30 @@ 4.0.0 com.eficode - devstack - 2.3.9-SNAPSHOT - jar + devstack-parent + 1.0 + pom DevStack A series of scripts for setting up common developer application suites + + + jirainstancemanager + jiraShortcuts + bitbucketinstancemanager + devstack + + + 11 11 3.0 [3.0,4.0) 2.4-M1-groovy-${groovy.major.version} - 2.0.3-SNAPSHOT-groovy-3.0 - 0.0.3-SNAPSHOT-groovy-3.0 - - - - - - org.codehaus.groovy - groovy-all - ${groovy.version} - provided - pom - - - org.codehaus.groovy - groovy-yaml - ${groovy.version} - - - org.codehaus.groovy - groovy-json - ${groovy.version} - - - - org.codehaus.groovy - groovy-ant - ${groovy.version} - - - org.spockframework - spock-core - ${spock-core.version} - test - - - - com.eficode.atlassian - jiraShortcuts - ${jiraShortcuts.version} - - - - - com.eficode.atlassian - bitbucketinstancemanager - ${bitbucketinstancemanager.version} - standalone - - - - - - org.slf4j - slf4j-api - 2.0.1 - - - org.slf4j - slf4j-simple - 2.0.0 - - - - org.apache.commons - commons-compress - 1.21 - - - - commons-io - commons-io - 2.11.0 - - - - com.eficode.atlassian - jirainstancemanager - 2.0.1-SNAPSHOT - - - - - - de.gesellix - docker-client - 2023-08-16T08-25-00 - - - - - - - org.junit.jupiter - junit-jupiter-api - 5.9.0 - test - - - org.junit.jupiter - junit-jupiter-engine - 5.9.0 - test - - - - - - - - - - eficode-github-jiraManagerRest - https://github.com/eficode/JiraInstanceManagerRest/raw/packages/repository/ - - - eficode-github-BitbucketInstanceManagerRest - https://github.com/eficode/BitbucketInstanceManagerRest/raw/packages/repository/ - - - eficode-github-JiraShortcuts - https://github.com/eficode/JiraShortcuts/raw/packages/repository/ - - - - - - - - - org.apache.maven.plugins - maven-source-plugin - 3.2.0 - - - attach-sources - generate-sources - - jar-no-fork - test-jar-no-fork - - - - - - org.codehaus.gmavenplus - gmavenplus-plugin - 2.0.0 - - - - addSources - addTestSources - compile - compileTests - - - - execute - - execute - - - - - - org.codehaus.groovy - groovy - ${groovy.version} - runtime - - - - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - 3.0.0-M7 - - - **/*Spec.class - **/*Test.java - - true - - - - - \ No newline at end of file diff --git a/src/main/groovy/com/eficode/devstack/container/impl/GroovyContainer.groovy b/src/main/groovy/com/eficode/devstack/container/impl/GroovyContainer.groovy deleted file mode 100644 index cf677e3..0000000 --- a/src/main/groovy/com/eficode/devstack/container/impl/GroovyContainer.groovy +++ /dev/null @@ -1,53 +0,0 @@ -package com.eficode.devstack.container.impl -import com.eficode.devstack.container.Container - - -class GroovyContainer implements Container{ - - String containerName = "groovy-container" - String containerMainPort = "" - String containerImage = "groovy" - String containerImageTag = "latest" - - - GroovyContainer(String dockerHost = "", String dockerCertPath = "") { - if (dockerHost && dockerCertPath) { - assert setupSecureRemoteConnection(dockerHost, dockerCertPath): "Error setting up secure remote docker connection" - } - } - - /** - * Runs a groovy script in the container - * @param scriptText Text of the script that should be run - * @param options Optional options to pass to the groovy process: ex: -Dorg.slf4j.simpleLogger.defaultLogLevel=trace - * @param arguments Optional arguments to pass to the script - * @param timeOutS - * @param containerUser What user to run the script as - * @return An arraylist containing console output from the script - */ - ArrayList runScriptInContainer(String scriptText,String options ="", String arguments = "", long timeOutS = 120, String containerUser = "groovy") { - assert replaceFileInContainer(scriptText, "/home/groovy/userScript.groovy") : "Error uploading script to container $id" - return runBashCommandInContainer("groovy $options /home/groovy/userScript.groovy $arguments 2>&1 | tee /var/log/userScript.log" , timeOutS, containerUser) - - } - - /** - * Creates a container that wont automatically terminate immediately - * @return container id - */ - String createSleepyContainer() { - return createContainer([],["/bin/bash" ,"-c" ,"trap \"exit\" SIGINT SIGTERM && tail -F /var/log/* /var/log/userScript.log"]) - } - - void useGroovy3() { - this.containerImageTag = "3.0-jdk11-jammy" - } - - void setGroovyVersion(String version) { - this.containerImageTag = "$version-jdk11-jammy" - } - - - - -} diff --git a/src/main/groovy/com/eficode/devstack/util/DockerClientDS.groovy b/src/main/groovy/com/eficode/devstack/util/DockerClientDS.groovy deleted file mode 100644 index bf03b05..0000000 --- a/src/main/groovy/com/eficode/devstack/util/DockerClientDS.groovy +++ /dev/null @@ -1,80 +0,0 @@ -package com.eficode.devstack.util - -import com.eficode.devstack.util.DockerClientDS -import de.gesellix.docker.client.DockerClientImpl -import de.gesellix.docker.client.EngineResponseContent -import de.gesellix.docker.engine.DockerClientConfig -import de.gesellix.docker.engine.DockerEnv -import de.gesellix.docker.remote.api.ExecConfig -import de.gesellix.docker.remote.api.ExecStartConfig -import de.gesellix.docker.remote.api.IdResponse -import de.gesellix.docker.remote.api.SystemInfo -import de.gesellix.docker.remote.api.core.Frame -import de.gesellix.docker.remote.api.core.StreamCallback -import org.slf4j.Logger -import org.slf4j.LoggerFactory - -import java.time.Duration - -import static java.net.Proxy.NO_PROXY - -class DockerClientDS extends DockerClientImpl { - - private final Logger log = LoggerFactory.getLogger(DockerClientDS) - - - DockerClientDS() { - super() - } - - DockerClientDS(String dockerHost) { - super(dockerHost) - } - - DockerClientDS(DockerEnv env, Proxy proxy = NO_PROXY) { - super(env, proxy) - } - - DockerClientDS(DockerClientConfig dockerClientConfig, Proxy proxy = NO_PROXY) { - super(dockerClientConfig, proxy) - } - - String getHost() { - return this.getEnv().dockerHost - } - String getCertPath() { - return this.getEnv().certPath - } - - - /** - * Returns the CPU architecture of the Docker engine - * @return ex: x86_64, aarch64 (arm) - */ - String getEngineArch() { - - SystemInfo info = info().content - - return info.architecture - - } - - - EngineResponseContent exec(String containerId, List command, StreamCallback callback, Duration timeout, ExecConfig execConfig) { - - log.info("docker exec '${containerId}' '${command}'") - - - - EngineResponseContent execCreateResult = createExec(containerId, execConfig) - String execId = execCreateResult.content.id - ExecStartConfig execStartConfig = new ExecStartConfig( - (execConfig.detachKeys ?: false) as Boolean, - execConfig.tty - ) - startExec(execId, execStartConfig, callback, timeout) - return execCreateResult - - } - -} From d20201f217450ba5521419c0d9ca9055ca188d17 Mon Sep 17 00:00:00 2001 From: Anders Lantz Date: Wed, 27 Sep 2023 09:33:33 +0200 Subject: [PATCH 2/2] DockerClientDS.groovy * Added to commit --- .../devstack/util/DockerClientDS.groovy | 202 ++++++++++++++++++ 1 file changed, 202 insertions(+) create mode 100644 devstack/src/main/groovy/com/eficode/devstack/util/DockerClientDS.groovy diff --git a/devstack/src/main/groovy/com/eficode/devstack/util/DockerClientDS.groovy b/devstack/src/main/groovy/com/eficode/devstack/util/DockerClientDS.groovy new file mode 100644 index 0000000..f12dec5 --- /dev/null +++ b/devstack/src/main/groovy/com/eficode/devstack/util/DockerClientDS.groovy @@ -0,0 +1,202 @@ +package com.eficode.devstack.util + +import com.eficode.devstack.container.impl.UbuntuContainer +import de.gesellix.docker.client.DockerClientImpl +import de.gesellix.docker.client.EngineResponseContent +import de.gesellix.docker.engine.DockerClientConfig +import de.gesellix.docker.engine.DockerEnv +import de.gesellix.docker.engine.EngineResponse +import de.gesellix.docker.remote.api.ContainerSummary +import de.gesellix.docker.remote.api.ExecConfig +import de.gesellix.docker.remote.api.ExecStartConfig +import de.gesellix.docker.remote.api.IdResponse +import de.gesellix.docker.remote.api.SystemInfo +import de.gesellix.docker.remote.api.Volume +import de.gesellix.docker.remote.api.VolumeCreateOptions +import de.gesellix.docker.remote.api.VolumeListResponse +import de.gesellix.docker.remote.api.core.Frame +import de.gesellix.docker.remote.api.core.StreamCallback +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +import java.time.Duration + +import static java.net.Proxy.NO_PROXY + +class DockerClientDS extends DockerClientImpl { + + private final Logger log = LoggerFactory.getLogger(DockerClientDS) + + + DockerClientDS() { + super() + } + + + DockerClientDS(String dockerHost) { + super(dockerHost) + } + + DockerClientDS(DockerEnv env, Proxy proxy = NO_PROXY) { + super(env, proxy) + } + + DockerClientDS(DockerClientConfig dockerClientConfig, Proxy proxy = NO_PROXY) { + super(dockerClientConfig, proxy) + } + + String getHost() { + return this.getEnv().dockerHost + } + + String getCertPath() { + return this.getEnv().certPath + } + + + /** + * Returns the CPU architecture of the Docker engine + * @return ex: x86_64, aarch64 (arm) + */ + String getEngineArch() { + + SystemInfo info = info().content + + return info.architecture + + } + + + ArrayList getVolumesWithLabel(String label) { + EngineResponseContent response = volumes("{\"label\":[\"$label\"]}") + + return response?.content?.volumes + } + + + ArrayList getVolumesWithName(String name) { + EngineResponseContent response = volumes("{\"name\":[\"$name\"]}") + + return response?.content?.volumes + } + + + ArrayList getContainersUsingVolume(Volume volume) { + + + EngineResponse response = ps(true, 1000, true, " {\"volume\":[\"${volume.name}\"]}") + + + ArrayList containers = response.content + return containers + + + } + + + EngineResponseContent createVolume(String name = null, Map labels = null, Map driverOpts = null) { + VolumeCreateOptions volumeOptions = new VolumeCreateOptions() + + //Transform Gstring to regular string + labels = labels?.collectEntries { [(it.key.toString()): it.value.toString()] } + driverOpts = driverOpts?.collectEntries { [(it.key.toString()): it.value.toString()] } + + volumeOptions.with { vol -> + vol.labels = labels + vol.name = name + vol.driverOpts = driverOpts + } + + + return createVolume(volumeOptions) + + + } + + + /** + * Clone a volume + * Clone is performed with a simple cp command, will fail if exotic file system is used + * The src volume must not be mounted to a container currently running + * @param srcVolumeName Name of the volume to copy + * @param destVolumeName Name of the destination volume. Must be unique and currently not created + * @param labels Labels of the new volume, null will leave them empty, [] will inherit from src + */ + Volume cloneVolume(String srcVolumeName, String destVolumeName, Map labels = null) { + + log.info("Cloning volume:" + srcVolumeName) + + ArrayList srcVolumes = getVolumesWithName(srcVolumeName).findAll { it.name == srcVolumeName } + assert srcVolumes.size() == 1: "Error finding source volume:" + srcVolumeName + + Volume srcVolume = srcVolumes.first() + + log.debug("\tSuccessfully identified volume to clone") + + ArrayList containersUsingSrc = getContainersUsingVolume(srcVolume) + + assert containersUsingSrc.empty || containersUsingSrc.findAll { it.state == "running" }.isEmpty(): "Source volume is currently connected to a running container" + + + Map destLabels + + switch (labels) { + + case null: + destLabels = null + break + case []: + destLabels = srcVolume.labels + break + default: + destLabels = labels + break + } + + + Volume destVolume = getVolumesWithName(destVolumeName).find { it.name == destVolumeName } + + assert destVolume == null: "Destination name is already in use" + + destVolume = createVolume(destVolumeName, destLabels, srcVolume.options)?.content + + assert destVolume: "Error creating Destination Volume" + log.info("\tCreated destination volume:" + destVolume.name) + + + UbuntuContainer ubuntuC = new UbuntuContainer(host, certPath) + + ubuntuC.prepareVolumeMount(srcVolume.name, "/srcVolume") + ubuntuC.prepareVolumeMount(destVolume.name, "/destVolume", false) + + ubuntuC.createSleepyContainer() + ubuntuC.startContainer() + ArrayList cmdOut = ubuntuC.runBashCommandInContainer("cp -av /srcVolume/* /destVolume/ && echo status: \$?") + ubuntuC.stopAndRemoveContainer() + assert cmdOut.any { it == "status: 0" }: "Error copying data to cloned volume" + log.info("\tSuccessfully copied data to volume") + + return destVolume + + + } + + + EngineResponseContent exec(String containerId, List command, StreamCallback callback, Duration timeout, ExecConfig execConfig) { + + + log.info("docker exec '${containerId}' '${command}'") + + + EngineResponseContent execCreateResult = createExec(containerId, execConfig) + String execId = execCreateResult.content.id + ExecStartConfig execStartConfig = new ExecStartConfig( + (execConfig.detachKeys ?: false) as Boolean, + execConfig.tty + ) + startExec(execId, execStartConfig, callback, timeout) + return execCreateResult + + } + +}