From 8decf1e23be4b1ea3828be7085f6a7add4e9858f Mon Sep 17 00:00:00 2001 From: Jared Bischof Date: Thu, 15 Aug 2013 13:03:45 -0500 Subject: [PATCH 01/33] Edits to make API url's show up correctly in return structures. --- src/MGRAST/cgi/api2.cgi | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/MGRAST/cgi/api2.cgi b/src/MGRAST/cgi/api2.cgi index 43dd13cd..124029a1 100644 --- a/src/MGRAST/cgi/api2.cgi +++ b/src/MGRAST/cgi/api2.cgi @@ -33,6 +33,13 @@ if ($abs !~ /\.cgi/) { } my $rest = $cgi->url(-path_info=>1); $rest =~ s/^.*$abs\/?//; +$rest =~ s/^\///; +$rest =~ s/^\d+//; +$rest =~ s/^beta//; +$rest =~ s/^\///; +$rest =~ s/^api2\.cgi//; +$rest =~ s/^\///; + my @rest_parameters = split m#/#, $rest; map {$rest[$_] =~ s#forwardslash#/#gi} (0 .. $#rest); @@ -177,11 +184,14 @@ if ($resource) { } # we are called without a resource, return API information else { - my @resource_objects = map { { 'name' => $_, 'url' => $cgi->url.'/'.$_ } } sort @$resources; + my $cgi_url = $cgi->url; + $cgi_url =~ s/^(.*)\/$/$1/; + $cgi_url =~ s/^(.*)\/api2.cgi$/$1/; + my @resource_objects = map { { 'name' => $_, 'url' => $cgi_url.'/'.$_ } } sort @$resources; my $content = { version => 1, service => 'MG-RAST', url => $cgi->url, - documentation => $cgi->url.'/api.html', + documentation => $cgi_url.'/api.html', description => "RESTful Metagenomics RAST object and resource API\nFor usage note that required parameters need to be passed as path parameters, optional parameters need to be query parameters. If an optional parameter has a list of option values, the first displayed will be used as default.", contact => 'mg-rast@mcs.anl.gov', resources => \@resource_objects }; From 18614d019059a69d9f3e964dfad493f587b965dc Mon Sep 17 00:00:00 2001 From: Jared Bischof Date: Thu, 15 Aug 2013 13:06:27 -0500 Subject: [PATCH 02/33] Edited documentation generator script to accept a site name as a command line option. --- src/WebApplication/scripts/api2html.pl | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/src/WebApplication/scripts/api2html.pl b/src/WebApplication/scripts/api2html.pl index 2e2e71bf..30cae982 100755 --- a/src/WebApplication/scripts/api2html.pl +++ b/src/WebApplication/scripts/api2html.pl @@ -12,17 +12,19 @@ sub usage { print "api2html.pl >>> create an HTML API documentation file from a REST API\n"; - print "api2html.pl -url -outfile \n"; + print "api2html.pl -url -site_name -outfile \n"; } # read in parameters -my $url = ''; -my $outfile = ''; +my $url = ''; +my $site_name = ''; +my $outfile = ''; GetOptions ( 'url=s' => \$url, + 'site_name=s' => \$site_name, 'outfile=s' => \$outfile ); -unless ($url and $outfile) { +unless ($url and $site_name and $outfile) { &usage(); exit 0; } @@ -58,7 +60,7 @@ sub usage { $structure->{resources} = $resources; # start the template -my $html = template_start(); +my $html = template_start($site_name); # build the navigation $html .= '
  • overview
    • '; @@ -70,7 +72,7 @@ sub usage {
    ~; -$html .= "

    ".$structure->{service}->{name}." Overview

    ".$structure->{service}->{description}."

    "; +$html .= "

    ".$site_name." Overview

    ".$structure->{service}->{description}."

    "; my %param_types = ( body => "

    This parameter must be passed in the message body.

    ", options => "

    This is an optional parameter and may be passed in the query string.

    ", @@ -113,6 +115,10 @@ sub usage { } } $html .= ""; + if (exists $req->{example}) { + $html .= "

    Example

      "; + $html .= "
    • ".$req->{example}[0]."
    • ".$req->{example}[1]."
    "; + } $html .= "

    Return Attributes

      "; # iterate over attributes foreach my $param (sort keys(%{$req->{attributes}})) { @@ -188,11 +194,12 @@ sub usage { exit; sub template_start { + my $site_name = shift; return qq~ - MG-RAST API + $site_name API @@ -202,7 +209,7 @@ sub template_start {
      - MG-RAST API documentation + $site_name API documentation
      From d0b7a72437780f6959a5847b94bc149650fcfa7c Mon Sep 17 00:00:00 2001 From: Jared Bischof Date: Thu, 15 Aug 2013 13:08:12 -0500 Subject: [PATCH 03/33] Edited Makefile to produce api documentation for MG-RAST and KBase on every make. --- Makefile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index e620a7db..87f0f3a4 100644 --- a/Makefile +++ b/Makefile @@ -51,8 +51,9 @@ OVERRIDES = \ .PHONY: lib clean purge # if you change all: make sure all: in standard.mk is consistent. +# api-doc target must be run last. -all: installdirs $(TOOL_HDR) lib +all: installdirs $(TOOL_HDR) lib api-doc # Use the PACKAGES macro to transform lib, bin, etc., targets # into package level dependencies, e.g., lib -> PkgA.lib, PkgB.lib @@ -67,7 +68,8 @@ purge: rm -rf $(TARGET) bin/*.r api-doc: - perl src/WebApplication/scripts/api2html.pl -url http://api.metagenomics.anl.gov -outfile site/CGI/Html/api.html + perl src/WebApplication/scripts/api2html.pl -url http://api.metagenomics.anl.gov/beta -site_name "MG-RAST" -outfile site/CGI/Html/api.html + perl src/WebApplication/scripts/api2html.pl -url http://kbase.us/services/communities/beta -site_name "Microbial Communities" -outfile site/CGI/Html/api.kbase.html ## # Targets to setup the expected directory structure for the From def701b8159d7c6f6034f0813d23bc19eefa81e1 Mon Sep 17 00:00:00 2001 From: Jared Bischof Date: Fri, 16 Aug 2013 13:53:43 -0500 Subject: [PATCH 04/33] Don't need to make api-doc target every time. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 87f0f3a4..8bbfcc69 100644 --- a/Makefile +++ b/Makefile @@ -53,7 +53,7 @@ OVERRIDES = \ # if you change all: make sure all: in standard.mk is consistent. # api-doc target must be run last. -all: installdirs $(TOOL_HDR) lib api-doc +all: installdirs $(TOOL_HDR) lib # Use the PACKAGES macro to transform lib, bin, etc., targets # into package level dependencies, e.g., lib -> PkgA.lib, PkgB.lib From 0cdeff203bc2bb4f5bbf82ae50dfa592fd0148c2 Mon Sep 17 00:00:00 2001 From: Jared Bischof Date: Mon, 3 Mar 2014 11:34:28 -0600 Subject: [PATCH 05/33] Adding this where clause makes queries for metadata verbosity=full calls execute at least 30 times faster, and should not break anything --- src/MGRAST/lib/Metadata.pm | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/MGRAST/lib/Metadata.pm b/src/MGRAST/lib/Metadata.pm index b58efcbd..3cdb3a3b 100644 --- a/src/MGRAST/lib/Metadata.pm +++ b/src/MGRAST/lib/Metadata.pm @@ -234,8 +234,9 @@ sub get_jobs_metadata_fast { my $dbh = $self->{_handle}->db_handle; my $key = $is_mgid ? 'metagenome_id' : 'job_id'; my $where = $is_mgid ? 'metagenome_id IN ('.join(',', map {"'$_'"} @$job_ids).')' : 'job_id IN ('.join(',', @$job_ids).')'; + my $where2= $is_mgid ? 'j.metagenome_id IN ('.join(',', map {"'$_'"} @$job_ids).')' : 'j.job_id IN ('.join(',', @$job_ids).')'; my $jobs = $dbh->selectall_arrayref("SELECT ".$key.",primary_project,sample,library,sequence_type,name,metagenome_id,file,file_checksum_raw FROM Job WHERE ".$where); - my $meth = $dbh->selectall_arrayref("SELECT j.".$key.", a.value FROM Job j, JobAttributes a WHERE j._id=a.job AND a.tag='sequencing_method_guess'"); + my $meth = $dbh->selectall_arrayref("SELECT j.".$key.", a.value FROM Job j, JobAttributes a WHERE j._id=a.job AND a.tag='sequencing_method_guess' AND ".$where2); my %pids = map { $_->[1], 1 } grep {$_->[1] && ($_->[1] =~ /\d+/)} @$jobs; my %sids = map { $_->[2], 1 } grep {$_->[2] && ($_->[2] =~ /\d+/)} @$jobs; my %lids = map { $_->[3], 1 } grep {$_->[3] && ($_->[3] =~ /\d+/)} @$jobs; From 90ffd6f60941e34e9660da647fc096df5fa6a90e Mon Sep 17 00:00:00 2001 From: Jared Bischof Date: Wed, 28 Jan 2015 11:51:01 -0600 Subject: [PATCH 06/33] Bug fix --- src/MGRAST/lib/resources/metagenome.pm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/MGRAST/lib/resources/metagenome.pm b/src/MGRAST/lib/resources/metagenome.pm index 5f7626cb..ff53393d 100644 --- a/src/MGRAST/lib/resources/metagenome.pm +++ b/src/MGRAST/lib/resources/metagenome.pm @@ -305,7 +305,7 @@ sub query { if ($self->user->has_star_right('view', 'metagenome')) { $solr_query_str .= "(status:private)"; } else { - if (scalar(%{$self->rights}) > 0) { + if (scalar(keys %{$self->rights}) > 0) { $solr_query_str .= "(status:private AND (".join(" OR ", map {'id:mgm'.$_} keys %{$self->rights})."))"; } else { $return_empty_set = 1; @@ -316,7 +316,7 @@ sub query { if ($self->user->has_star_right('view', 'metagenome')) { $solr_query_str .= "(status:*)"; } else { - if (scalar(%{$self->rights}) > 0) { + if (scalar(keys %{$self->rights}) > 0) { $solr_query_str .= "((status:public) OR (status:private AND (".join(" OR ", map {'id:mgm'.$_} keys %{$self->rights}).")))"; } else { $solr_query_str .= '(status:public)'; From 512abd2df82a941e5509b63ef8909a5077f6c329 Mon Sep 17 00:00:00 2001 From: Wolfgang Gerlach Date: Mon, 8 Jun 2015 11:55:00 -0500 Subject: [PATCH 07/33] httpd.conf --- conf/httpd.conf | 507 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 507 insertions(+) create mode 100644 conf/httpd.conf diff --git a/conf/httpd.conf b/conf/httpd.conf new file mode 100644 index 00000000..a4b72e87 --- /dev/null +++ b/conf/httpd.conf @@ -0,0 +1,507 @@ +# +# This is the main Apache HTTP server configuration file. It contains the +# configuration directives that give the server its instructions. +# See for detailed information. +# In particular, see +# +# for a discussion of each configuration directive. +# +# Do NOT simply read the instructions in here without understanding +# what they do. They're here only as hints or reminders. If you are unsure +# consult the online docs. You have been warned. +# +# Configuration and logfile names: If the filenames you specify for many +# of the server's control files begin with "/" (or "drive:/" for Win32), the +# server will use that explicit path. If the filenames do *not* begin +# with "/", the value of ServerRoot is prepended -- so "logs/access_log" +# with ServerRoot set to "/usr/local/apache2" will be interpreted by the +# server as "/usr/local/apache2/logs/access_log", whereas "/logs/access_log" +# will be interpreted as '/logs/access_log'. + +# +# ServerRoot: The top of the directory tree under which the server's +# configuration, error, and log files are kept. +# +# Do not add a slash at the end of the directory path. If you point +# ServerRoot at a non-local disk, be sure to specify a local disk on the +# Mutex directive, if file-based mutexes are used. If you wish to share the +# same ServerRoot for multiple httpd daemons, you will need to change at +# least PidFile. +# +ServerRoot "/usr/local/apache2" + +Include /MG-RAST/conf/api.metagenomics.conf + +# +# Mutex: Allows you to set the mutex mechanism and mutex file directory +# for individual mutexes, or change the global defaults +# +# Uncomment and change the directory if mutexes are file-based and the default +# mutex file directory is not on a local disk or is not appropriate for some +# other reason. +# +# Mutex default:logs + +# +# Listen: Allows you to bind Apache to specific IP addresses and/or +# ports, instead of the default. See also the +# directive. +# +# Change this to Listen on specific IP addresses as shown below to +# prevent Apache from glomming onto all bound IP addresses. +# +#Listen 12.34.56.78:80 +Listen 80 + +# +# Dynamic Shared Object (DSO) Support +# +# To be able to use the functionality of a module which was built as a DSO you +# have to place corresponding `LoadModule' lines at this location so the +# directives contained in it are actually available _before_ they are used. +# Statically compiled modules (those listed by `httpd -l') do not need +# to be loaded here. +# +# Example: +# LoadModule foo_module modules/mod_foo.so +# +LoadModule authn_file_module modules/mod_authn_file.so +#LoadModule authn_dbm_module modules/mod_authn_dbm.so +#LoadModule authn_anon_module modules/mod_authn_anon.so +#LoadModule authn_dbd_module modules/mod_authn_dbd.so +#LoadModule authn_socache_module modules/mod_authn_socache.so +LoadModule authn_core_module modules/mod_authn_core.so +LoadModule authz_host_module modules/mod_authz_host.so +LoadModule authz_groupfile_module modules/mod_authz_groupfile.so +LoadModule authz_user_module modules/mod_authz_user.so +#LoadModule authz_dbm_module modules/mod_authz_dbm.so +#LoadModule authz_owner_module modules/mod_authz_owner.so +#LoadModule authz_dbd_module modules/mod_authz_dbd.so +LoadModule authz_core_module modules/mod_authz_core.so +#LoadModule authnz_ldap_module modules/mod_authnz_ldap.so +LoadModule access_compat_module modules/mod_access_compat.so +LoadModule auth_basic_module modules/mod_auth_basic.so +#LoadModule auth_form_module modules/mod_auth_form.so +#LoadModule auth_digest_module modules/mod_auth_digest.so +#LoadModule allowmethods_module modules/mod_allowmethods.so +#LoadModule file_cache_module modules/mod_file_cache.so +#LoadModule cache_module modules/mod_cache.so +#LoadModule cache_disk_module modules/mod_cache_disk.so +#LoadModule cache_socache_module modules/mod_cache_socache.so +#LoadModule socache_shmcb_module modules/mod_socache_shmcb.so +#LoadModule socache_dbm_module modules/mod_socache_dbm.so +#LoadModule socache_memcache_module modules/mod_socache_memcache.so +#LoadModule macro_module modules/mod_macro.so +#LoadModule dbd_module modules/mod_dbd.so +#LoadModule dumpio_module modules/mod_dumpio.so +#LoadModule buffer_module modules/mod_buffer.so +#LoadModule ratelimit_module modules/mod_ratelimit.so +LoadModule reqtimeout_module modules/mod_reqtimeout.so +#LoadModule ext_filter_module modules/mod_ext_filter.so +#LoadModule request_module modules/mod_request.so +#LoadModule include_module modules/mod_include.so +LoadModule filter_module modules/mod_filter.so +#LoadModule substitute_module modules/mod_substitute.so +#LoadModule sed_module modules/mod_sed.so +#LoadModule deflate_module modules/mod_deflate.so +LoadModule mime_module modules/mod_mime.so +#LoadModule ldap_module modules/mod_ldap.so +LoadModule log_config_module modules/mod_log_config.so +#LoadModule log_debug_module modules/mod_log_debug.so +#LoadModule logio_module modules/mod_logio.so +LoadModule env_module modules/mod_env.so +#LoadModule expires_module modules/mod_expires.so +LoadModule headers_module modules/mod_headers.so +#LoadModule unique_id_module modules/mod_unique_id.so +LoadModule setenvif_module modules/mod_setenvif.so +LoadModule version_module modules/mod_version.so +#LoadModule remoteip_module modules/mod_remoteip.so +#LoadModule proxy_module modules/mod_proxy.so +#LoadModule proxy_connect_module modules/mod_proxy_connect.so +#LoadModule proxy_ftp_module modules/mod_proxy_ftp.so +#LoadModule proxy_http_module modules/mod_proxy_http.so +#LoadModule proxy_fcgi_module modules/mod_proxy_fcgi.so +#LoadModule proxy_scgi_module modules/mod_proxy_scgi.so +#LoadModule proxy_wstunnel_module modules/mod_proxy_wstunnel.so +#LoadModule proxy_ajp_module modules/mod_proxy_ajp.so +#LoadModule proxy_balancer_module modules/mod_proxy_balancer.so +#LoadModule proxy_express_module modules/mod_proxy_express.so +#LoadModule session_module modules/mod_session.so +#LoadModule session_cookie_module modules/mod_session_cookie.so +#LoadModule session_crypto_module modules/mod_session_crypto.so +#LoadModule session_dbd_module modules/mod_session_dbd.so +#LoadModule slotmem_shm_module modules/mod_slotmem_shm.so +#LoadModule ssl_module modules/mod_ssl.so +#LoadModule lbmethod_byrequests_module modules/mod_lbmethod_byrequests.so +#LoadModule lbmethod_bytraffic_module modules/mod_lbmethod_bytraffic.so +#LoadModule lbmethod_bybusyness_module modules/mod_lbmethod_bybusyness.so +#LoadModule lbmethod_heartbeat_module modules/mod_lbmethod_heartbeat.so +LoadModule unixd_module modules/mod_unixd.so +#LoadModule dav_module modules/mod_dav.so +LoadModule status_module modules/mod_status.so +LoadModule autoindex_module modules/mod_autoindex.so +#LoadModule info_module modules/mod_info.so +#LoadModule cgid_module modules/mod_cgid.so +#LoadModule dav_fs_module modules/mod_dav_fs.so +#LoadModule vhost_alias_module modules/mod_vhost_alias.so +#LoadModule negotiation_module modules/mod_negotiation.so +LoadModule dir_module modules/mod_dir.so +#LoadModule actions_module modules/mod_actions.so +#LoadModule speling_module modules/mod_speling.so +#LoadModule userdir_module modules/mod_userdir.so +LoadModule alias_module modules/mod_alias.so +#LoadModule rewrite_module modules/mod_rewrite.so + + +# +# If you wish httpd to run as a different user or group, you must run +# httpd as root initially and it will switch. +# +# User/Group: The name (or #number) of the user/group to run httpd as. +# It is usually good practice to create a dedicated user and group for +# running httpd, as with most system services. +# +User daemon +Group daemon + + + +# 'Main' server configuration +# +# The directives in this section set up the values used by the 'main' +# server, which responds to any requests that aren't handled by a +# definition. These values also provide defaults for +# any containers you may define later in the file. +# +# All of these directives may appear inside containers, +# in which case these default settings will be overridden for the +# virtual host being defined. +# + +# +# ServerAdmin: Your address, where problems with the server should be +# e-mailed. This address appears on some server-generated pages, such +# as error documents. e.g. admin@your-domain.com +# +ServerAdmin you@example.com + +# +# ServerName gives the name and port that the server uses to identify itself. +# This can often be determined automatically, but we recommend you specify +# it explicitly to prevent problems during startup. +# +# If your host doesn't have a registered DNS name, enter its IP address here. +# +#ServerName www.example.com:80 + +# +# Deny access to the entirety of your server's filesystem. You must +# explicitly permit access to web content directories in other +# blocks below. +# + + AllowOverride none + Require all denied + + +# +# Note that from this point forward you must specifically allow +# particular features to be enabled - so if something's not working as +# you might expect, make sure that you have specifically enabled it +# below. +# + +# +# DocumentRoot: The directory out of which you will serve your +# documents. By default, all requests are taken from this directory, but +# symbolic links and aliases may be used to point to other locations. +# +DocumentRoot "/usr/local/apache2/htdocs" + + # + # Possible values for the Options directive are "None", "All", + # or any combination of: + # Indexes Includes FollowSymLinks SymLinksifOwnerMatch ExecCGI MultiViews + # + # Note that "MultiViews" must be named *explicitly* --- "Options All" + # doesn't give it to you. + # + # The Options directive is both complicated and important. Please see + # http://httpd.apache.org/docs/2.4/mod/core.html#options + # for more information. + # + Options Indexes FollowSymLinks + + # + # AllowOverride controls what directives may be placed in .htaccess files. + # It can be "All", "None", or any combination of the keywords: + # AllowOverride FileInfo AuthConfig Limit + # + AllowOverride None + + # + # Controls who can get stuff from this server. + # + Require all granted + + +# +# DirectoryIndex: sets the file that Apache will serve if a directory +# is requested. +# + + DirectoryIndex index.html + + +# +# The following lines prevent .htaccess and .htpasswd files from being +# viewed by Web clients. +# + + Require all denied + + +# +# ErrorLog: The location of the error log file. +# If you do not specify an ErrorLog directive within a +# container, error messages relating to that virtual host will be +# logged here. If you *do* define an error logfile for a +# container, that host's errors will be logged there and not here. +# +ErrorLog /proc/self/fd/2 + +# +# LogLevel: Control the number of messages logged to the error_log. +# Possible values include: debug, info, notice, warn, error, crit, +# alert, emerg. +# +LogLevel warn + + + # + # The following directives define some format nicknames for use with + # a CustomLog directive (see below). + # + LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined + LogFormat "%h %l %u %t \"%r\" %>s %b" common + + + # You need to enable mod_logio.c to use %I and %O + LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %I %O" combinedio + + + # + # The location and format of the access logfile (Common Logfile Format). + # If you do not define any access logfiles within a + # container, they will be logged here. Contrariwise, if you *do* + # define per- access logfiles, transactions will be + # logged therein and *not* in this file. + # + CustomLog /proc/self/fd/1 common + + # + # If you prefer a logfile with access, agent, and referer information + # (Combined Logfile Format) you can use the following directive. + # + #CustomLog "logs/access_log" combined + + + + # + # Redirect: Allows you to tell clients about documents that used to + # exist in your server's namespace, but do not anymore. The client + # will make a new request for the document at its new location. + # Example: + # Redirect permanent /foo http://www.example.com/bar + + # + # Alias: Maps web paths into filesystem paths and is used to + # access content that does not live under the DocumentRoot. + # Example: + # Alias /webpath /full/filesystem/path + # + # If you include a trailing / on /webpath then the server will + # require it to be present in the URL. You will also likely + # need to provide a section to allow access to + # the filesystem path. + + # + # ScriptAlias: This controls which directories contain server scripts. + # ScriptAliases are essentially the same as Aliases, except that + # documents in the target directory are treated as applications and + # run by the server when requested rather than as documents sent to the + # client. The same rules about trailing "/" apply to ScriptAlias + # directives as to Alias. + # + ScriptAlias /cgi-bin/ "/usr/local/apache2/cgi-bin/" + + + + + # + # ScriptSock: On threaded servers, designate the path to the UNIX + # socket used to communicate with the CGI daemon of mod_cgid. + # + #Scriptsock cgisock + + +# +# "/usr/local/apache2/cgi-bin" should be changed to whatever your ScriptAliased +# CGI directory exists, if you have that configured. +# + + AllowOverride None + Options None + Require all granted + + + + # + # TypesConfig points to the file containing the list of mappings from + # filename extension to MIME-type. + # + TypesConfig conf/mime.types + + # + # AddType allows you to add to or override the MIME configuration + # file specified in TypesConfig for specific file types. + # + #AddType application/x-gzip .tgz + # + # AddEncoding allows you to have certain browsers uncompress + # information on the fly. Note: Not all browsers support this. + # + #AddEncoding x-compress .Z + #AddEncoding x-gzip .gz .tgz + # + # If the AddEncoding directives above are commented-out, then you + # probably should define those extensions to indicate media types: + # + AddType application/x-compress .Z + AddType application/x-gzip .gz .tgz + + # + # AddHandler allows you to map certain file extensions to "handlers": + # actions unrelated to filetype. These can be either built into the server + # or added with the Action directive (see below) + # + # To use CGI scripts outside of ScriptAliased directories: + # (You will also need to add "ExecCGI" to the "Options" directive.) + # + #AddHandler cgi-script .cgi + + # For type maps (negotiated resources): + #AddHandler type-map var + + # + # Filters allow you to process content before it is sent to the client. + # + # To parse .shtml files for server-side includes (SSI): + # (You will also need to add "Includes" to the "Options" directive.) + # + #AddType text/html .shtml + #AddOutputFilter INCLUDES .shtml + + +# +# The mod_mime_magic module allows the server to use various hints from the +# contents of the file itself to determine its type. The MIMEMagicFile +# directive tells the module where the hint definitions are located. +# +#MIMEMagicFile conf/magic + +# +# Customizable error responses come in three flavors: +# 1) plain text 2) local redirects 3) external redirects +# +# Some examples: +#ErrorDocument 500 "The server made a boo boo." +#ErrorDocument 404 /missing.html +#ErrorDocument 404 "/cgi-bin/missing_handler.pl" +#ErrorDocument 402 http://www.example.com/subscription_info.html +# + +# +# MaxRanges: Maximum number of Ranges in a request before +# returning the entire resource, or one of the special +# values 'default', 'none' or 'unlimited'. +# Default setting is to accept 200 Ranges. +#MaxRanges unlimited + +# +# EnableMMAP and EnableSendfile: On systems that support it, +# memory-mapping or the sendfile syscall may be used to deliver +# files. This usually improves server performance, but must +# be turned off when serving from networked-mounted +# filesystems or if support for these functions is otherwise +# broken on your system. +# Defaults: EnableMMAP On, EnableSendfile Off +# +#EnableMMAP off +#EnableSendfile on + +# Supplemental configuration +# +# The configuration files in the conf/extra/ directory can be +# included to add extra features or to modify the default configuration of +# the server, or you may simply copy their contents here and change as +# necessary. + +# Server-pool management (MPM specific) +#Include conf/extra/httpd-mpm.conf + +# Multi-language error messages +#Include conf/extra/httpd-multilang-errordoc.conf + +# Fancy directory listings +#Include conf/extra/httpd-autoindex.conf + +# Language settings +#Include conf/extra/httpd-languages.conf + +# User home directories +#Include conf/extra/httpd-userdir.conf + +# Real-time info on requests and configuration +#Include conf/extra/httpd-info.conf + +# Virtual hosts +#Include conf/extra/httpd-vhosts.conf + +# Local access to the Apache HTTP Server Manual +#Include conf/extra/httpd-manual.conf + +# Distributed authoring and versioning (WebDAV) +#Include conf/extra/httpd-dav.conf + +# Various default settings +#Include conf/extra/httpd-default.conf + +# Configure mod_proxy_html to understand HTML4/XHTML1 + +Include conf/extra/proxy-html.conf + + +# Secure (SSL/TLS) connections +#Include conf/extra/httpd-ssl.conf +# +# Note: The following must must be present to support +# starting without SSL on platforms with no /dev/random equivalent +# but a statically compiled-in mod_ssl. +# + +SSLRandomSeed startup builtin +SSLRandomSeed connect builtin + +# +# uncomment out the below to deal with user agents that deliberately +# violate open standards by misusing DNT (DNT *must* be a specific +# end-user choice) +# +# +#BrowserMatch "MSIE 10.0;" bad_DNT +# +# +#RequestHeader unset DNT env=bad_DNT +# + From e026ec82811d28d7ba82f204ee8565297cfe7b1c Mon Sep 17 00:00:00 2001 From: Wolfgang Gerlach Date: Mon, 8 Jun 2015 13:56:33 -0500 Subject: [PATCH 08/33] enable rewrite module --- conf/httpd.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/httpd.conf b/conf/httpd.conf index a4b72e87..50acc3f7 100644 --- a/conf/httpd.conf +++ b/conf/httpd.conf @@ -150,7 +150,7 @@ LoadModule dir_module modules/mod_dir.so #LoadModule speling_module modules/mod_speling.so #LoadModule userdir_module modules/mod_userdir.so LoadModule alias_module modules/mod_alias.so -#LoadModule rewrite_module modules/mod_rewrite.so +LoadModule rewrite_module modules/mod_rewrite.so # From e839303ef5e75b8156eceed7a5f895fff4827570 Mon Sep 17 00:00:00 2001 From: Wolfgang Gerlach Date: Mon, 8 Jun 2015 17:17:00 -0500 Subject: [PATCH 09/33] enable cgi --- conf/httpd.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/httpd.conf b/conf/httpd.conf index 50acc3f7..2172e743 100644 --- a/conf/httpd.conf +++ b/conf/httpd.conf @@ -141,7 +141,7 @@ LoadModule unixd_module modules/mod_unixd.so LoadModule status_module modules/mod_status.so LoadModule autoindex_module modules/mod_autoindex.so #LoadModule info_module modules/mod_info.so -#LoadModule cgid_module modules/mod_cgid.so +LoadModule cgid_module modules/mod_cgid.so #LoadModule dav_fs_module modules/mod_dav_fs.so #LoadModule vhost_alias_module modules/mod_vhost_alias.so #LoadModule negotiation_module modules/mod_negotiation.so From 23bf1abade49919fe6c86b03a241b98cd3b491cd Mon Sep 17 00:00:00 2001 From: Wolfgang Gerlach Date: Mon, 8 Jun 2015 17:33:37 -0500 Subject: [PATCH 10/33] Dockerfile --- docker/Dockerfile | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 docker/Dockerfile diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 00000000..4f89d563 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,24 @@ +FROM httpd:2.4.12 + +RUN apt-get update && apt-get install -y \ + git-core \ + liburi-perl \ + libwww-perl + + +RUN mkdir -p /var/log/httpd/api.metagenomics/ && \ + cd / && \ + git clone -b api https://github.com/MG-RAST/MG-RAST.git && \ + cd /MG-RAST && \ + make + +RUN mkdir -p /sites/1/ && \ + cd /sites/1/ && \ + ln -s /MG-RAST/ + +# Configuration in mounted directory +RUN cd /MG-RAST/conf && ln -s /api-server/Conf.pm + + +# Execute: +# /usr/local/apache2/bin/httpd -DFOREGROUND -f /MG-RAST/conf/httpd.conf \ No newline at end of file From b6fa0d83216dbdcde3fa7fb140774e5d361316a7 Mon Sep 17 00:00:00 2001 From: Wolfgang Gerlach Date: Tue, 9 Jun 2015 11:38:17 -0500 Subject: [PATCH 11/33] pipeline --- docker/Dockerfile | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docker/Dockerfile b/docker/Dockerfile index 4f89d563..13fa2ba0 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -12,6 +12,11 @@ RUN mkdir -p /var/log/httpd/api.metagenomics/ && \ cd /MG-RAST && \ make + +#RUN apt-get install -y PIPELINE_DEPENDENCIES HERE <------------------ + +RUN cd / && git clone https://github.com/MG-RAST/pipeline.git + RUN mkdir -p /sites/1/ && \ cd /sites/1/ && \ ln -s /MG-RAST/ @@ -20,5 +25,7 @@ RUN mkdir -p /sites/1/ && \ RUN cd /MG-RAST/conf && ln -s /api-server/Conf.pm + + # Execute: # /usr/local/apache2/bin/httpd -DFOREGROUND -f /MG-RAST/conf/httpd.conf \ No newline at end of file From 30846deb06c13748a4a16641840979e5b7c1b48a Mon Sep 17 00:00:00 2001 From: Travis Harrison Date: Tue, 9 Jun 2015 13:57:25 -0500 Subject: [PATCH 12/33] update dependencies --- docker/Dockerfile | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 13fa2ba0..0ea53ab4 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,9 +1,27 @@ FROM httpd:2.4.12 +# MG-RAST dependencies RUN apt-get update && apt-get install -y \ git-core \ + libpq-dev + perl-modules \ liburi-perl \ - libwww-perl + libwww-perl \ + libjson-perl \ + libdbi-perl \ + libdbd-mysql-perl \ + libdbd-pg-perl \ + libdigest-md5-perl \ + libfile-slurp-perl \ + libhtml-strip-perl \ + liblist-moreutils-perl \ + libcache-memcached-perl \ + libhtml-template-perl \ + libdigest-md5-perl \ + libdigest-md5-file-perl \ + libdatetime-perl \ + liblist-allutils-perl \ + libposix-strptime-perl RUN mkdir -p /var/log/httpd/api.metagenomics/ && \ @@ -12,10 +30,16 @@ RUN mkdir -p /var/log/httpd/api.metagenomics/ && \ cd /MG-RAST && \ make +# pipeline dependencies +RUN apt-get install -y \ + python-dev \ + python-pip \ + libtemplate-perl -#RUN apt-get install -y PIPELINE_DEPENDENCIES HERE <------------------ +RUN pip install gspread xlrd openpyxl lepl RUN cd / && git clone https://github.com/MG-RAST/pipeline.git +RUN cp /pipeline/lib/StreamingUpload.pm /MG-RAST/site/lib/StreamingUpload.pm RUN mkdir -p /sites/1/ && \ cd /sites/1/ && \ @@ -23,8 +47,7 @@ RUN mkdir -p /sites/1/ && \ # Configuration in mounted directory RUN cd /MG-RAST/conf && ln -s /api-server/Conf.pm - - +RUN cd /pipeline/conf && ln -s /api-server/PipelineAWE_Conf.pm # Execute: From 9e8dc9972fc969bb3bf334beab216b4e3eac2a6a Mon Sep 17 00:00:00 2001 From: Travis Harrison Date: Tue, 9 Jun 2015 14:06:49 -0500 Subject: [PATCH 13/33] local version of streamingupload --- src/MGRAST/lib/StreamingUpload.pm | 66 +++++++++++++++++++++++++++++++ src/MGRAST/lib/resources/inbox.pm | 4 +- 2 files changed, 68 insertions(+), 2 deletions(-) create mode 100644 src/MGRAST/lib/StreamingUpload.pm diff --git a/src/MGRAST/lib/StreamingUpload.pm b/src/MGRAST/lib/StreamingUpload.pm new file mode 100644 index 00000000..861d69ba --- /dev/null +++ b/src/MGRAST/lib/StreamingUpload.pm @@ -0,0 +1,66 @@ +package StreamingUpload; + +use strict; +use warnings; +our $VERSION = '0.01'; + +use Carp (); +use HTTP::Request; + +sub new { + my($class, $method, $uri, %args) = @_; + + my $headers = $args{headers}; + if ($headers) { + if (ref $headers eq 'HASH') { + $headers = +[ %{ $headers } ]; + } + } + + my $req = HTTP::Request->new($method, $uri, $headers); + _set_content($req, \%args); + $req; +} + +sub _set_content { + my($req, $args) = @_; + + if ($args->{content}) { + $req->content($args->{content}); + } elsif ($args->{callback} && ref($args->{callback}) eq 'CODE') { + $req->content($args->{callback}); + } elsif ($args->{path} || $args->{fh}) { + my $fh; + if ($args->{fh}) { + $fh = $args->{fh}; + } else { + open $fh, '<', $args->{path} or Carp::croak "$args->{path}: $!"; + } + my $chunk_size = $args->{chunk_size} || 4096; + $req->content(sub { + my $len = read($fh, my $buf, $chunk_size); + return unless $len; + return $buf; + }); + } +} + +sub slurp { + my(undef, $req) = @_; + my $content_ref = $req->content_ref; + $content_ref = ${ $content_ref } if ref ${ $content_ref }; + + my $content; + if (ref($content_ref) eq 'CODE') { + while (1) { + my $buf = $content_ref->(); + last unless defined $buf; + $content .= $buf; + } + } else { + $content = ${ $content_ref }; + } + $content; +} + +1; diff --git a/src/MGRAST/lib/resources/inbox.pm b/src/MGRAST/lib/resources/inbox.pm index 3ffdc49d..4de25ed5 100644 --- a/src/MGRAST/lib/resources/inbox.pm +++ b/src/MGRAST/lib/resources/inbox.pm @@ -5,7 +5,7 @@ use warnings; no warnings('once'); use POSIX qw(strftime); -use HTTP::Request::StreamingUpload; +use StreamingUpload; use HTTP::Headers; use LWP::UserAgent; use File::Basename; @@ -595,7 +595,7 @@ sub upload_file { my $response = undef; my $io_handle = $fh->handle; eval { - my $post = HTTP::Request::StreamingUpload->new( + my $post = StreamingUpload->new( POST => $Conf::shock_url.'/node', fh => $io_handle, headers => HTTP::Headers->new( From 3fad7358cfa5b184ed1989bbb6bc10ae8b48810f Mon Sep 17 00:00:00 2001 From: Travis Harrison Date: Tue, 9 Jun 2015 14:07:22 -0500 Subject: [PATCH 14/33] update dockerfile --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 0ea53ab4..25b190b5 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -39,7 +39,7 @@ RUN apt-get install -y \ RUN pip install gspread xlrd openpyxl lepl RUN cd / && git clone https://github.com/MG-RAST/pipeline.git -RUN cp /pipeline/lib/StreamingUpload.pm /MG-RAST/site/lib/StreamingUpload.pm + RUN mkdir -p /sites/1/ && \ cd /sites/1/ && \ From 1736b620d6a0b2e4d7899ab5978fd555c0bb21d8 Mon Sep 17 00:00:00 2001 From: Travis Harrison Date: Tue, 9 Jun 2015 14:17:35 -0500 Subject: [PATCH 15/33] fix dockerfile --- docker/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 25b190b5..58785b38 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -3,7 +3,7 @@ FROM httpd:2.4.12 # MG-RAST dependencies RUN apt-get update && apt-get install -y \ git-core \ - libpq-dev + libpq-dev \ perl-modules \ liburi-perl \ libwww-perl \ @@ -46,8 +46,8 @@ RUN mkdir -p /sites/1/ && \ ln -s /MG-RAST/ # Configuration in mounted directory -RUN cd /MG-RAST/conf && ln -s /api-server/Conf.pm -RUN cd /pipeline/conf && ln -s /api-server/PipelineAWE_Conf.pm +RUN cd /MG-RAST/conf && ln -s /api-server/Conf.pm && \ + cd /pipeline/conf && ln -s /api-server/PipelineAWE_Conf.pm # Execute: From b55b2ec513b5364bfdb951c3f9cb97e205087a79 Mon Sep 17 00:00:00 2001 From: Travis Harrison Date: Tue, 9 Jun 2015 14:26:32 -0500 Subject: [PATCH 16/33] Update Dockerfile --- docker/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 58785b38..a50f471d 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -4,6 +4,7 @@ FROM httpd:2.4.12 RUN apt-get update && apt-get install -y \ git-core \ libpq-dev \ + make \ perl-modules \ liburi-perl \ libwww-perl \ @@ -51,4 +52,4 @@ RUN cd /MG-RAST/conf && ln -s /api-server/Conf.pm && \ # Execute: -# /usr/local/apache2/bin/httpd -DFOREGROUND -f /MG-RAST/conf/httpd.conf \ No newline at end of file +# /usr/local/apache2/bin/httpd -DFOREGROUND -f /MG-RAST/conf/httpd.conf From 115b2ba2859d2576e38540bf30880cd34735ed20 Mon Sep 17 00:00:00 2001 From: Travis Harrison Date: Tue, 9 Jun 2015 14:37:15 -0500 Subject: [PATCH 17/33] added m5nr download --- docker/Dockerfile | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 58785b38..0028d911 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -49,6 +49,18 @@ RUN mkdir -p /sites/1/ && \ RUN cd /MG-RAST/conf && ln -s /api-server/Conf.pm && \ cd /pipeline/conf && ln -s /api-server/PipelineAWE_Conf.pm +# m5nr blast files +RUN mkdir -p /m5nr/20100309 && \ + cd /m5nr/20100309 && \ + wget ftp://ftp.metagenomics.anl.gov/data/MD5nr/20100309/md5nr.blast.tgz && \ + tar zxvf md5nr.blast.tgz && \ + rm md5nr.blast.tgz + +RUN mkdir -p /m5nr/20131215 && \ + cd /m5nr/20131215 && \ + wget ftp://ftp.metagenomics.anl.gov/data/MD5nr/20131215/md5nr.blast.tgz && \ + tar zxvf md5nr.blast.tgz && \ + rm md5nr.blast.tgz # Execute: -# /usr/local/apache2/bin/httpd -DFOREGROUND -f /MG-RAST/conf/httpd.conf \ No newline at end of file +# /usr/local/apache2/bin/httpd -DFOREGROUND -f /MG-RAST/conf/httpd.conf From 61f87dd05b07d8974878bc97e01d65e0ab7baf22 Mon Sep 17 00:00:00 2001 From: Travis Harrison Date: Tue, 9 Jun 2015 14:39:37 -0500 Subject: [PATCH 18/33] Update Dockerfile --- docker/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/Dockerfile b/docker/Dockerfile index 66ec4439..a15fcdea 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -5,6 +5,7 @@ RUN apt-get update && apt-get install -y \ git-core \ libpq-dev \ make \ + wget \ perl-modules \ liburi-perl \ libwww-perl \ From 227c04d2278f4bda1fbe73cbef8f0f910da8998f Mon Sep 17 00:00:00 2001 From: Travis Harrison Date: Tue, 9 Jun 2015 14:46:56 -0500 Subject: [PATCH 19/33] Update Dockerfile --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index a15fcdea..6f99aab5 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -51,7 +51,7 @@ RUN mkdir -p /sites/1/ && \ RUN cd /MG-RAST/conf && ln -s /api-server/Conf.pm && \ cd /pipeline/conf && ln -s /api-server/PipelineAWE_Conf.pm -# m5nr blast files +# m5nr blast files - 2.6 GB & 11.7 GB RUN mkdir -p /m5nr/20100309 && \ cd /m5nr/20100309 && \ wget ftp://ftp.metagenomics.anl.gov/data/MD5nr/20100309/md5nr.blast.tgz && \ From 889a3def43dbc9ae17cc361371b23c724a9e5482 Mon Sep 17 00:00:00 2001 From: Travis Harrison Date: Thu, 11 Jun 2015 13:36:15 -0500 Subject: [PATCH 20/33] seperate script for downloading m5nr blast files --- bin/download_m5nr_blast.sh | 27 +++++++++++++++++++++++++++ docker/Dockerfile | 22 +++++++--------------- 2 files changed, 34 insertions(+), 15 deletions(-) create mode 100644 bin/download_m5nr_blast.sh diff --git a/bin/download_m5nr_blast.sh b/bin/download_m5nr_blast.sh new file mode 100644 index 00000000..d12d1660 --- /dev/null +++ b/bin/download_m5nr_blast.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# set a default values +# m5nr blast files - 2.6 GB & 11.7 GB +M5NR_VERSIONS="20100309 20131215" +TARGET="/m5nr" + +for i in $M5NR_VERSIONS; do + M5NR_VERSION=$i + VERSION_DIR=$TARGET/$M5NR_VERSION + URL=ftp://ftp.metagenomics.anl.gov/data/MD5nr/${M5NR_VERSION}/md5nr.blast.tgz + + echo "" + echo "TARGET = $TARGET" + echo "M5NR_VERSION = $M5NR_VERSION" + echo "URL = $URL" + echo "" + + if [ -d ${VERSION_DIR} ]; then + echo "Files already exist, not downloading" + else + echo "Downloading files" + mkdir -p ${VERSION_DIR} + curl -s "${URL}" | tar -zxvf - -C ${VERSION_DIR} + fi +done +exit 0 \ No newline at end of file diff --git a/docker/Dockerfile b/docker/Dockerfile index 6f99aab5..10e4431b 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -5,7 +5,7 @@ RUN apt-get update && apt-get install -y \ git-core \ libpq-dev \ make \ - wget \ + curl \ perl-modules \ liburi-perl \ libwww-perl \ @@ -48,21 +48,13 @@ RUN mkdir -p /sites/1/ && \ ln -s /MG-RAST/ # Configuration in mounted directory -RUN cd /MG-RAST/conf && ln -s /api-server/Conf.pm && \ - cd /pipeline/conf && ln -s /api-server/PipelineAWE_Conf.pm +RUN cd /MG-RAST/conf && ln -s /api-server/conf/Conf.pm && \ + cd /pipeline/conf && ln -s /api-server/conf/PipelineAWE_Conf.pm -# m5nr blast files - 2.6 GB & 11.7 GB -RUN mkdir -p /m5nr/20100309 && \ - cd /m5nr/20100309 && \ - wget ftp://ftp.metagenomics.anl.gov/data/MD5nr/20100309/md5nr.blast.tgz && \ - tar zxvf md5nr.blast.tgz && \ - rm md5nr.blast.tgz - -RUN mkdir -p /m5nr/20131215 && \ - cd /m5nr/20131215 && \ - wget ftp://ftp.metagenomics.anl.gov/data/MD5nr/20131215/md5nr.blast.tgz && \ - tar zxvf md5nr.blast.tgz && \ - rm md5nr.blast.tgz +# m5nr blast files in mounted dir +RUN mkdir -p /m5nr && \ + ln -s /api-server/data/20100309 /m5nr/20100309 && \ + ln -s /api-server/data/20131215 /m5nr/20131215 # Execute: # /usr/local/apache2/bin/httpd -DFOREGROUND -f /MG-RAST/conf/httpd.conf From da099d941d139651342428b157f9b886291c4427 Mon Sep 17 00:00:00 2001 From: Wolfgang Gerlach Date: Mon, 15 Jun 2015 10:21:15 -0500 Subject: [PATCH 21/33] fix path --- conf/httpd.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/httpd.conf b/conf/httpd.conf index 2172e743..8c949aa0 100644 --- a/conf/httpd.conf +++ b/conf/httpd.conf @@ -30,7 +30,7 @@ # ServerRoot "/usr/local/apache2" -Include /MG-RAST/conf/api.metagenomics.conf +Include /api-server/api.metagenomics.conf # # Mutex: Allows you to set the mutex mechanism and mutex file directory From 0f32122f87b72b4503af1d6643733501082eb542 Mon Sep 17 00:00:00 2001 From: Wolfgang Gerlach Date: Mon, 15 Jun 2015 10:29:53 -0500 Subject: [PATCH 22/33] fix paths --- conf/httpd.conf | 2 +- docker/Dockerfile | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/conf/httpd.conf b/conf/httpd.conf index 8c949aa0..623d3f1e 100644 --- a/conf/httpd.conf +++ b/conf/httpd.conf @@ -30,7 +30,7 @@ # ServerRoot "/usr/local/apache2" -Include /api-server/api.metagenomics.conf +Include /api-server-conf/api.metagenomics.conf # # Mutex: Allows you to set the mutex mechanism and mutex file directory diff --git a/docker/Dockerfile b/docker/Dockerfile index 10e4431b..8a9d99b6 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -48,8 +48,8 @@ RUN mkdir -p /sites/1/ && \ ln -s /MG-RAST/ # Configuration in mounted directory -RUN cd /MG-RAST/conf && ln -s /api-server/conf/Conf.pm && \ - cd /pipeline/conf && ln -s /api-server/conf/PipelineAWE_Conf.pm +RUN cd /MG-RAST/conf && ln -s /api-server-conf/Conf.pm && \ + cd /pipeline/conf && ln -s /api-server-conf/PipelineAWE_Conf.pm # m5nr blast files in mounted dir RUN mkdir -p /m5nr && \ From 19abf4d163415cb5b3e63d39f375b0576c7292ce Mon Sep 17 00:00:00 2001 From: Travis Harrison Date: Mon, 15 Jun 2015 10:44:18 -0500 Subject: [PATCH 23/33] Update Dockerfile --- docker/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 8a9d99b6..f55890e0 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -23,7 +23,8 @@ RUN apt-get update && apt-get install -y \ libdigest-md5-file-perl \ libdatetime-perl \ liblist-allutils-perl \ - libposix-strptime-perl + libposix-strptime-perl \ + libuuid-tiny-perl RUN mkdir -p /var/log/httpd/api.metagenomics/ && \ From d6ecbeb44a1a484a72ef0d4ec2313615cadba338 Mon Sep 17 00:00:00 2001 From: Wolfgang Gerlach Date: Mon, 15 Jun 2015 11:01:28 -0500 Subject: [PATCH 24/33] fix path --- docker/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index f55890e0..0426db37 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -54,8 +54,8 @@ RUN cd /MG-RAST/conf && ln -s /api-server-conf/Conf.pm && \ # m5nr blast files in mounted dir RUN mkdir -p /m5nr && \ - ln -s /api-server/data/20100309 /m5nr/20100309 && \ - ln -s /api-server/data/20131215 /m5nr/20131215 + ln -s /api-server-data/20100309 /m5nr/20100309 && \ + ln -s /api-server-data/20131215 /m5nr/20131215 # Execute: # /usr/local/apache2/bin/httpd -DFOREGROUND -f /MG-RAST/conf/httpd.conf From 3e357f9503cf72e1a3e6532e5310dff9d70a89d4 Mon Sep 17 00:00:00 2001 From: Wolfgang Gerlach Date: Mon, 15 Jun 2015 11:11:45 -0500 Subject: [PATCH 25/33] documentation --- conf/README.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 conf/README.md diff --git a/conf/README.md b/conf/README.md new file mode 100644 index 00000000..01bffe1e --- /dev/null +++ b/conf/README.md @@ -0,0 +1,22 @@ + + +API server +---------- + + +Build image: +```bash +export TAG=`date +"%Y%m%d.%H%M"` +docker build --force-rm --no-cache --rm -t mgrast/api:${TAG} https://raw.githubusercontent.com/MG-RAST/MG-RAST/api/docker/Dockerfile +``` + +Get config: (private mcs git repo) +```bash +if cd /home/core/mgrast-config; then git pull; else cd /home/core/ ; git clone git@git.mcs.anl.gov:mgrast-config.git ; fi +``` + + +Start container: +```bash +docker run -t -i --name api -v ~/mgrast-config/services/api-server:/api-server-conf -v /media/ephemeral/api-server-data:/api-server-data -p 80:80 mgrast/api:${TAG} /usr/local/apache2/bin/httpd -DFOREGROUND -f /MG-RAST/conf/httpd.conf +``` \ No newline at end of file From cc30a6f154997b792251ae9e5fb300a01b96632a Mon Sep 17 00:00:00 2001 From: Wolfgang Gerlach Date: Mon, 15 Jun 2015 11:13:14 -0500 Subject: [PATCH 26/33] abs path --- conf/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/README.md b/conf/README.md index 01bffe1e..0a3511c6 100644 --- a/conf/README.md +++ b/conf/README.md @@ -18,5 +18,5 @@ if cd /home/core/mgrast-config; then git pull; else cd /home/core/ ; git clone g Start container: ```bash -docker run -t -i --name api -v ~/mgrast-config/services/api-server:/api-server-conf -v /media/ephemeral/api-server-data:/api-server-data -p 80:80 mgrast/api:${TAG} /usr/local/apache2/bin/httpd -DFOREGROUND -f /MG-RAST/conf/httpd.conf +docker run -t -i --name api -v /home/core/mgrast-config/services/api-server:/api-server-conf -v /media/ephemeral/api-server-data:/api-server-data -p 80:80 mgrast/api:${TAG} /usr/local/apache2/bin/httpd -DFOREGROUND -f /MG-RAST/conf/httpd.conf ``` \ No newline at end of file From dd84377b623c7a16b65165567917ed997779b1b4 Mon Sep 17 00:00:00 2001 From: Wolfgang Gerlach Date: Mon, 15 Jun 2015 11:26:40 -0500 Subject: [PATCH 27/33] rename path --- conf/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/README.md b/conf/README.md index 0a3511c6..a689a658 100644 --- a/conf/README.md +++ b/conf/README.md @@ -18,5 +18,5 @@ if cd /home/core/mgrast-config; then git pull; else cd /home/core/ ; git clone g Start container: ```bash -docker run -t -i --name api -v /home/core/mgrast-config/services/api-server:/api-server-conf -v /media/ephemeral/api-server-data:/api-server-data -p 80:80 mgrast/api:${TAG} /usr/local/apache2/bin/httpd -DFOREGROUND -f /MG-RAST/conf/httpd.conf -``` \ No newline at end of file +docker run -t -i --name api -v /home/core/mgrast-config/services/api-server:/api-server-conf -v /media/ephemeral/api-server-data:/m5nr -p 80:80 mgrast/api:${TAG} /usr/local/apache2/bin/httpd -DFOREGROUND -f /MG-RAST/conf/httpd.conf +``` From 37d9f29091b0db3e9db55220fc7c7be99dd36722 Mon Sep 17 00:00:00 2001 From: Wolfgang Gerlach Date: Mon, 15 Jun 2015 11:30:42 -0500 Subject: [PATCH 28/33] download data instructions --- conf/README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/conf/README.md b/conf/README.md index a689a658..2381b65e 100644 --- a/conf/README.md +++ b/conf/README.md @@ -15,6 +15,11 @@ Get config: (private mcs git repo) if cd /home/core/mgrast-config; then git pull; else cd /home/core/ ; git clone git@git.mcs.anl.gov:mgrast-config.git ; fi ``` +Download data +```bash +docker run -t -i --name api -v /media/ephemeral/api-server-data:/m5nr mgrast/api:${TAG} /MG-RAST/bin/download_m5nr_blast.sh +docker rm api +``` Start container: ```bash From 2139763e92a028e048b70131608d672a8bc7fb83 Mon Sep 17 00:00:00 2001 From: Wolfgang Gerlach Date: Mon, 15 Jun 2015 11:37:42 -0500 Subject: [PATCH 29/33] exec --- bin/download_m5nr_blast.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 bin/download_m5nr_blast.sh diff --git a/bin/download_m5nr_blast.sh b/bin/download_m5nr_blast.sh old mode 100644 new mode 100755 From b853f696be034e83419ea694714ac31aac260947 Mon Sep 17 00:00:00 2001 From: Wolfgang Gerlach Date: Mon, 15 Jun 2015 16:20:59 -0500 Subject: [PATCH 30/33] Update README.md --- conf/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/README.md b/conf/README.md index 2381b65e..4236a5e3 100644 --- a/conf/README.md +++ b/conf/README.md @@ -10,7 +10,7 @@ export TAG=`date +"%Y%m%d.%H%M"` docker build --force-rm --no-cache --rm -t mgrast/api:${TAG} https://raw.githubusercontent.com/MG-RAST/MG-RAST/api/docker/Dockerfile ``` -Get config: (private mcs git repo) +Get config: (private mcs git repo, for details see fleet unit) ```bash if cd /home/core/mgrast-config; then git pull; else cd /home/core/ ; git clone git@git.mcs.anl.gov:mgrast-config.git ; fi ``` @@ -23,5 +23,5 @@ docker rm api Start container: ```bash -docker run -t -i --name api -v /home/core/mgrast-config/services/api-server:/api-server-conf -v /media/ephemeral/api-server-data:/m5nr -p 80:80 mgrast/api:${TAG} /usr/local/apache2/bin/httpd -DFOREGROUND -f /MG-RAST/conf/httpd.conf +docker run -t -i --name api -v /home/core/mgrast-config/services/api-server/.postgresql/:/root/.postgresql/:ro -v /home/core/mgrast-config/services/api-server:/api-server-conf -v /media/ephemeral/api-server-data:/m5nr -p 80:80 mgrast/api:${TAG} /usr/local/apache2/bin/httpd -DFOREGROUND -f /MG-RAST/conf/httpd.conf ``` From 875e8b370e0d42e18b2a70d4f052a854c01b661a Mon Sep 17 00:00:00 2001 From: Wolfgang Gerlach Date: Tue, 16 Jun 2015 12:42:13 -0500 Subject: [PATCH 31/33] root for cert --- conf/httpd.conf | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/conf/httpd.conf b/conf/httpd.conf index 623d3f1e..c50738ad 100644 --- a/conf/httpd.conf +++ b/conf/httpd.conf @@ -161,8 +161,10 @@ LoadModule rewrite_module modules/mod_rewrite.so # It is usually good practice to create a dedicated user and group for # running httpd, as with most system services. # -User daemon -Group daemon +#User daemon +#Group daemon +User root +Group root From 58a661785d5221a54158282ef05f71d9a9947668 Mon Sep 17 00:00:00 2001 From: Wolfgang Gerlach Date: Tue, 16 Jun 2015 12:53:58 -0500 Subject: [PATCH 32/33] reverse changes, use daemon --- conf/httpd.conf | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/conf/httpd.conf b/conf/httpd.conf index c50738ad..623d3f1e 100644 --- a/conf/httpd.conf +++ b/conf/httpd.conf @@ -161,10 +161,8 @@ LoadModule rewrite_module modules/mod_rewrite.so # It is usually good practice to create a dedicated user and group for # running httpd, as with most system services. # -#User daemon -#Group daemon -User root -Group root +User daemon +Group daemon From ba54cdba85329a67c15142926ed71851b699ea9a Mon Sep 17 00:00:00 2001 From: Travis Harrison Date: Thu, 25 Jun 2015 12:36:04 -0500 Subject: [PATCH 33/33] restructure docker files to allow multiple --- docker/Dockerfile | 61 ----------------------------------------------- 1 file changed, 61 deletions(-) delete mode 100644 docker/Dockerfile diff --git a/docker/Dockerfile b/docker/Dockerfile deleted file mode 100644 index 0426db37..00000000 --- a/docker/Dockerfile +++ /dev/null @@ -1,61 +0,0 @@ -FROM httpd:2.4.12 - -# MG-RAST dependencies -RUN apt-get update && apt-get install -y \ - git-core \ - libpq-dev \ - make \ - curl \ - perl-modules \ - liburi-perl \ - libwww-perl \ - libjson-perl \ - libdbi-perl \ - libdbd-mysql-perl \ - libdbd-pg-perl \ - libdigest-md5-perl \ - libfile-slurp-perl \ - libhtml-strip-perl \ - liblist-moreutils-perl \ - libcache-memcached-perl \ - libhtml-template-perl \ - libdigest-md5-perl \ - libdigest-md5-file-perl \ - libdatetime-perl \ - liblist-allutils-perl \ - libposix-strptime-perl \ - libuuid-tiny-perl - - -RUN mkdir -p /var/log/httpd/api.metagenomics/ && \ - cd / && \ - git clone -b api https://github.com/MG-RAST/MG-RAST.git && \ - cd /MG-RAST && \ - make - -# pipeline dependencies -RUN apt-get install -y \ - python-dev \ - python-pip \ - libtemplate-perl - -RUN pip install gspread xlrd openpyxl lepl - -RUN cd / && git clone https://github.com/MG-RAST/pipeline.git - - -RUN mkdir -p /sites/1/ && \ - cd /sites/1/ && \ - ln -s /MG-RAST/ - -# Configuration in mounted directory -RUN cd /MG-RAST/conf && ln -s /api-server-conf/Conf.pm && \ - cd /pipeline/conf && ln -s /api-server-conf/PipelineAWE_Conf.pm - -# m5nr blast files in mounted dir -RUN mkdir -p /m5nr && \ - ln -s /api-server-data/20100309 /m5nr/20100309 && \ - ln -s /api-server-data/20131215 /m5nr/20131215 - -# Execute: -# /usr/local/apache2/bin/httpd -DFOREGROUND -f /MG-RAST/conf/httpd.conf