diff --git a/dockerfiles/api/Dockerfile b/dockerfiles/api/Dockerfile index 0426db37..e4542906 100644 --- a/dockerfiles/api/Dockerfile +++ b/dockerfiles/api/Dockerfile @@ -1,3 +1,5 @@ +# MG-RAST API + FROM httpd:2.4.12 # MG-RAST dependencies @@ -6,6 +8,7 @@ RUN apt-get update && apt-get install -y \ libpq-dev \ make \ curl \ + r-base \ perl-modules \ liburi-perl \ libwww-perl \ @@ -40,10 +43,8 @@ RUN apt-get install -y \ libtemplate-perl RUN pip install gspread xlrd openpyxl lepl - RUN cd / && git clone https://github.com/MG-RAST/pipeline.git - RUN mkdir -p /sites/1/ && \ cd /sites/1/ && \ ln -s /MG-RAST/ @@ -58,4 +59,4 @@ RUN mkdir -p /m5nr && \ ln -s /api-server-data/20131215 /m5nr/20131215 # Execute: -# /usr/local/apache2/bin/httpd -DFOREGROUND -f /MG-RAST/conf/httpd.conf +# /usr/local/apache2/bin/httpd -DFOREGROUND -f /api-server-conf/httpd.conf diff --git a/dockerfiles/web/Dockerfile b/dockerfiles/web/Dockerfile index ff850f56..f3fb6519 100644 --- a/dockerfiles/web/Dockerfile +++ b/dockerfiles/web/Dockerfile @@ -64,6 +64,7 @@ RUN cd / && \ # certificates need to be in daemon home directory RUN ln -s /config/postgresql/ /usr/sbin/.postgresql +# docker run -t -i --name web -v /home/core/mgrast-config/services/web_v3/:/config -p 80:80 httpd:2.4.12 bash # Start -#bash -c "/bin/chmod 600 /config/postgresql/* ; /bin/chown daemon:daemon /config/postgresql/* ; /usr/local/apache2/bin/apachectl -f /config/httpd.conf -D FOREGROUND" +# bash -c "/bin/chmod 600 /config/postgresql/*; /bin/chown daemon:daemon /config/postgresql/*; /usr/local/apache2/bin/apachectl -f /config/httpd.conf -D FOREGROUND" diff --git a/src/MGRAST/Schema/m5nr_copy.cql.tt b/src/MGRAST/Schema/m5nr_copy.cql.tt new file mode 100644 index 00000000..04aa3ed9 --- /dev/null +++ b/src/MGRAST/Schema/m5nr_copy.cql.tt @@ -0,0 +1,17 @@ + +use m5nr_v[% version %]; + +COPY ontologies (source, name, level1, level2, level3, level4) FROM '[% data_dir %]/m5nr_v[% version %].ontology.all'; +COPY ont_level1 (source, level1, name) FROM '[% data_dir %]/m5nr_v[% version %].ontology.level1'; +COPY ont_level2 (source, level2, name) FROM '[% data_dir %]/m5nr_v[% version %].ontology.level2'; +COPY ont_level3 (source, level3, name) FROM '[% data_dir %]/m5nr_v[% version %].ontology.level3'; +COPY ont_level4 (source, level4, name) FROM '[% data_dir %]/m5nr_v[% version %].ontology.level4'; + +COPY organisms_ncbi (name, tax_domain, tax_phylum, tax_class, tax_order, tax_family, tax_genus, tax_species, ncbi_tax_id) FROM '[% data_dir %]/m5nr_v[% version %].taxonomy.all'; +COPY tax_domain (tax_domain, name) FROM '[% data_dir %]/m5nr_v[% version %].taxonomy.domain'; +COPY tax_phylum (tax_phylum, name) FROM '[% data_dir %]/m5nr_v[% version %].taxonomy.phylum'; +COPY tax_class (tax_class, name) FROM '[% data_dir %]/m5nr_v[% version %].taxonomy.class'; +COPY tax_order (tax_order, name) FROM '[% data_dir %]/m5nr_v[% version %].taxonomy.order'; +COPY tax_family (tax_family, name) FROM '[% data_dir %]/m5nr_v[% version %].taxonomy.family'; +COPY tax_genus (tax_genus, name) FROM '[% data_dir %]/m5nr_v[% version %].taxonomy.genus'; +COPY tax_species (tax_species, name) FROM '[% data_dir %]/m5nr_v[% version %].taxonomy.species'; diff --git a/src/MGRAST/Schema/m5nr_v4.cql b/src/MGRAST/Schema/m5nr_table.cql.tt similarity index 60% rename from src/MGRAST/Schema/m5nr_v4.cql rename to src/MGRAST/Schema/m5nr_table.cql.tt index 02d87d36..04f94a3c 100644 --- a/src/MGRAST/Schema/m5nr_v4.cql +++ b/src/MGRAST/Schema/m5nr_table.cql.tt @@ -1,10 +1,10 @@ -CREATE KEYSPACE IF NOT EXISTS m5nr_v1 - WITH REPLICATION = { 'class' : 'SimpleStrategy', 'replication_factor' : 2 }; +CREATE KEYSPACE IF NOT EXISTS m5nr_v[% version %] + WITH REPLICATION = { 'class' : 'SimpleStrategy', 'replication_factor' : [% replication %] }; -use m5nr_v1; +use m5nr_v[% version %]; -CREATE TABLE IF NOT EXISTS md5_id_annotation ( +CREATE TABLE IF NOT EXISTS id_annotation ( id int, source text, md5 text, @@ -16,20 +16,18 @@ CREATE TABLE IF NOT EXISTS md5_id_annotation ( organism list, PRIMARY KEY (id, source) ); -COPY md5_id_annotation (id, source, md5, is_protein, single, lca, accession, function, organism) FROM '/mnt/src/m5nr_v1.annotation.id'; CREATE TABLE IF NOT EXISTS md5_annotation ( md5 text, source text, is_protein boolean, single text, - lca text, + lca list, accession list, function list, organism list, PRIMARY KEY (md5, source) ); -COPY md5_annotation (md5, source, is_protein, single, lca, accession, function, organism) FROM '/mnt/src/m5nr_v1.annotation.md5'; CREATE TABLE IF NOT EXISTS ontologies ( source text, @@ -40,7 +38,6 @@ CREATE TABLE IF NOT EXISTS ontologies ( level4 text, PRIMARY KEY (source, name) ); -COPY ontologies (source, name, level1, level2, level3, level4) FROM '/mnt/src/m5nr_v1.ontology.all'; CREATE TABLE IF NOT EXISTS ont_level1 ( source text, @@ -48,7 +45,6 @@ CREATE TABLE IF NOT EXISTS ont_level1 ( name text, PRIMARY KEY (source, level1, name) ); -COPY ont_level1 (source, level1, name) FROM '/mnt/src/m5nr_v1.ontology.level1'; CREATE TABLE IF NOT EXISTS ont_level2 ( source text, @@ -56,7 +52,6 @@ CREATE TABLE IF NOT EXISTS ont_level2 ( name text, PRIMARY KEY (source, level2, name) ); -COPY ont_level2 (source, level2, name) FROM '/mnt/src/m5nr_v1.ontology.level2'; CREATE TABLE IF NOT EXISTS ont_level3 ( source text, @@ -64,7 +59,6 @@ CREATE TABLE IF NOT EXISTS ont_level3 ( name text, PRIMARY KEY (source, level3, name) ); -COPY ont_level3 (source, level3, name) FROM '/mnt/src/m5nr_v1.ontology.level3'; CREATE TABLE IF NOT EXISTS ont_level4 ( source text, @@ -72,7 +66,6 @@ CREATE TABLE IF NOT EXISTS ont_level4 ( name text, PRIMARY KEY (source, level4, name) ); -COPY ont_level4 (source, level4, name) FROM '/mnt/src/m5nr_v1.ontology.level4'; CREATE TABLE IF NOT EXISTS organisms_ncbi ( name text, @@ -86,53 +79,45 @@ CREATE TABLE IF NOT EXISTS organisms_ncbi ( ncbi_tax_id int, PRIMARY KEY (name) ); -COPY organisms_ncbi (name, tax_domain, tax_phylum, tax_class, tax_order, tax_family, tax_genus, tax_species, ncbi_tax_id) FROM '/mnt/src/m5nr_v1.taxonomy.all'; CREATE TABLE IF NOT EXISTS tax_domain ( tax_domain text, name text, PRIMARY KEY (tax_domain, name) ); -COPY tax_domain (tax_domain, name) FROM '/mnt/src/m5nr_v1.taxonomy.domain'; CREATE TABLE IF NOT EXISTS tax_phylum ( tax_phylum text, name text, PRIMARY KEY (tax_phylum, name) ); -COPY tax_phylum (tax_phylum, name) FROM '/mnt/src/m5nr_v1.taxonomy.phylum'; CREATE TABLE IF NOT EXISTS tax_class ( tax_class text, name text, PRIMARY KEY (tax_class, name) ); -COPY tax_class (tax_class, name) FROM '/mnt/src/m5nr_v1.taxonomy.class'; CREATE TABLE IF NOT EXISTS tax_order ( tax_order text, name text, PRIMARY KEY (tax_order, name) ); -COPY tax_order (tax_order, name) FROM '/mnt/src/m5nr_v1.taxonomy.order'; CREATE TABLE IF NOT EXISTS tax_family ( tax_family text, name text, PRIMARY KEY (tax_family, name) ); -COPY tax_family (tax_family, name) FROM '/mnt/src/m5nr_v1.taxonomy.family'; CREATE TABLE IF NOT EXISTS tax_genus ( tax_genus text, name text, PRIMARY KEY (tax_genus, name) ); -COPY tax_genus (tax_genus, name) FROM '/mnt/src/m5nr_v1.taxonomy.genus'; CREATE TABLE IF NOT EXISTS tax_species ( tax_species text, name text, PRIMARY KEY (tax_species, name) ); -COPY tax_species (tax_species, name) FROM '/mnt/src/m5nr_v1.taxonomy.species'; diff --git a/src/MGRAST/Schema/mgrast_analysis_v4.cql b/src/MGRAST/Schema/mgrast_analysis_v4.cql index 7d4bde93..9376df40 100644 --- a/src/MGRAST/Schema/mgrast_analysis_v4.cql +++ b/src/MGRAST/Schema/mgrast_analysis_v4.cql @@ -12,9 +12,15 @@ CREATE TYPE source_info ( CREATE TYPE md5_info ( abundance int, + exp_avg float, + ident_avg float, + len_avg float, + exp_stdv float, + ident_stdv float, + len_stdv float, seek bigint, length int, - annotation map> + is_protein boolean ); CREATE TYPE lca_info ( @@ -42,6 +48,7 @@ INSERT INTO job_test (version, job, exp_avg, md5, data) VALUES (1, 20, 60, 'foo' INSERT INTO job_test (version, job, exp_avg, md5, data) VALUES (1, 20, 70, 'foo', 'hello world'); INSERT INTO job_test (version, job, exp_avg, md5, data) VALUES (1, 20, 70, 'bar', 'hello world'); +### version 1 ### CREATE TABLE IF NOT EXISTS job_md5s ( version int, job int, @@ -70,6 +77,7 @@ CREATE TABLE IF NOT EXISTS job_lcas ( # see: http://docs.datastax.com/en/cql/3.1/cql/cql_using/use-slice-partition.html +### version 2 ### CREATE TABLE IF NOT EXISTS job_md5s ( version int, job int, @@ -107,3 +115,17 @@ CREATE TABLE IF NOT EXISTS job_lcas ( PRIMARY KEY ((version, job), exp_avg, ident_avg, len_avg, lca) ); +### version 3 ### +CREATE TABLE IF NOT EXISTS job_md5s ( + version int, + job int, + source text, + md5 text, + data frozen, + accession list, + function list, + organism list, + PRIMARY KEY ((version, job), source, md5) +); + + diff --git a/src/MGRAST/html/MGRAST-frontpage.tmpl b/src/MGRAST/html/MGRAST-frontpage.tmpl index 2c08eee2..76077232 100644 --- a/src/MGRAST/html/MGRAST-frontpage.tmpl +++ b/src/MGRAST/html/MGRAST-frontpage.tmpl @@ -8,7 +8,7 @@
- + diff --git a/src/MGRAST/html/MGRAST.tmpl b/src/MGRAST/html/MGRAST.tmpl index c6914940..d449bc10 100644 --- a/src/MGRAST/html/MGRAST.tmpl +++ b/src/MGRAST/html/MGRAST.tmpl @@ -14,7 +14,7 @@
- + diff --git a/src/MGRAST/html/js/MetagenomeSearch.js b/src/MGRAST/html/js/MetagenomeSearch.js index 77e3d759..747ec77c 100644 --- a/src/MGRAST/html/js/MetagenomeSearch.js +++ b/src/MGRAST/html/js/MetagenomeSearch.js @@ -1,4 +1,4 @@ -var api_url = 'http://api.metagenomics.anl.gov/1/metagenome?verbosity=mixs&'; +var api_url = RetinaConfig ? RetinaConfig.mgrast_api + "/metagenome?verbosity=mixs&" : 'http://api.metagenomics.anl.gov/1/metagenome?verbosity=mixs&'; var datastore = {}; var result = 'result'; // div where results are to be displayed var saved_params = {}; diff --git a/src/MGRAST/lib/JobDB/Metadata.pm b/src/MGRAST/lib/JobDB/Metadata.pm index dd6b2bb3..01741ca3 100644 --- a/src/MGRAST/lib/JobDB/Metadata.pm +++ b/src/MGRAST/lib/JobDB/Metadata.pm @@ -2,6 +2,7 @@ package JobDB::Metadata; use strict; use warnings; +use Conf; 1; @@ -130,7 +131,7 @@ sub get_data { $obj->{id} = "mgm".$job->{metagenome_id}; $obj->{about} = "mobedac metagenome"; $obj->{name} = $job->{name}; - $obj->{url} = 'http://metagenomics.anl.gov/linkin.cgi?id='.$obj->{id}; + $obj->{url} = $Conf::cgi_url.'linkin.cgi?id='.$obj->{id}; $obj->{version} = $job->{server_version}; $obj->{creation} = $job->{created_on}; # $obj->{sample} = $job->sample->{ID}; diff --git a/src/MGRAST/lib/WebPage/Analysis.pm b/src/MGRAST/lib/WebPage/Analysis.pm index 6e57ac68..1a558cae 100644 --- a/src/MGRAST/lib/WebPage/Analysis.pm +++ b/src/MGRAST/lib/WebPage/Analysis.pm @@ -6603,8 +6603,8 @@ sub selectable_metagenomes { push @{$collections->{$name}}, [ $pj->{metagenome_id}, $pj->{name} ]; } foreach my $coll ( sort keys %$collections ) { - if ( @{$collections->{$coll}} == 0 ) { next; } - push(@$colls, { label => $coll." [".scalar(@{$collections->{$coll}})."]", value => join('||', map { $_->[0]."##".$_->[1] } @{$collections->{$coll}}) }); + if ( (! $coll) || (! $collections->{$coll}) || (@{$collections->{$coll}} == 0) ) { next; } + push(@$colls, { label => $coll." [".scalar(@{$collections->{$coll}})."]", value => join('||', map { ($_->[0] || "")."##".($_->[1] || "") } @{$collections->{$coll}}) }); } } diff --git a/src/MGRAST/lib/WebPage/DownloadMetagenome.pm b/src/MGRAST/lib/WebPage/DownloadMetagenome.pm index da4eb29f..698277bb 100644 --- a/src/MGRAST/lib/WebPage/DownloadMetagenome.pm +++ b/src/MGRAST/lib/WebPage/DownloadMetagenome.pm @@ -73,7 +73,7 @@ sub init { $self->data('default', $default); # api info for download - $self->data('api', "http://api.metagenomics.anl.gov"); + $self->data('api', $Conf::api_url || "http://api.metagenomics.anl.gov"); # get to metagenome using the metagenome ID if ( $cgi->param('metagenome') ) { diff --git a/src/MGRAST/lib/WebPage/Home.pm b/src/MGRAST/lib/WebPage/Home.pm index c133c4af..c6610911 100644 --- a/src/MGRAST/lib/WebPage/Home.pm +++ b/src/MGRAST/lib/WebPage/Home.pm @@ -58,7 +58,7 @@ sub output { if ($is_old) { $content .= qq~~; + ~; } $content .= "
"; @@ -117,7 +117,7 @@ function forward_to_search (e) { $content .= "
".$register."
"; $content .= "
".$contact."
"; $content .= "
".$help."
"; - $content .= "
".$upload."
"; + $content .= "
".$upload."
"; $content .= "
".$news."
"; $content .= ""; $content .= "
"; @@ -176,13 +176,7 @@ END $content .= "

cite MG-RAST

cite MG-RAST API

"; - #$content .= "

The Metagenomics RAST server - A public resource for the automatic phylogenetic and functional analysis of metagenomes F. Meyer, D. Paarmann, M. D'Souza, R. Olson , E. M. Glass, M. Kubal, T. Paczian , A. Rodriguez , R. Stevens, A. Wilke, J. Wilkening, R. A. Edwards
BMC Bioinformatics 2008, 9:386 -#, [article]

"; - - #$content .= "

powered by

"; - my $logos = ""; - # news - logos return $content; } @@ -198,36 +192,6 @@ sub speedometer { my ($speed, $mileage, $trip, $togo) = split(/\t/, $line); $content .= "
pipeline status
$speedbp/sspeed
$tripMbplast 30 days
$togoMbpin queue
$mileageMbptotal
"; - - # my @trip1 = split(//, $trip); -# while (scalar(@trip1) < 7) { -# unshift(@trip1, "0"); -# } -# $trip = '';# style="margin-left: 36px;" -# foreach my $t (@trip1) { -# $trip .= ""; -# } -# $trip .= '
$t
'; -# my @togo1 = split(//, $togo); -# while (scalar(@togo1) < 7) { -# unshift(@togo1, "0"); -# } -# $togo = '';# style="margin-left: 36px;" -# foreach my $t (@togo1) { -# $togo .= ""; -# } -# $togo .= '
$t
'; -# my @mileage1 = split(//, $mileage); -# while (scalar(@mileage1) < 7) { -# unshift(@mileage1, "0"); -# } -# $mileage = ''; -# foreach my $t (@mileage1) { -# $mileage .= ""; -# } -# $mileage .= '
$t
'; - -# $content .= "
$speedbp/sspeed
$tripMbplast 30 days
$togoMbpin queue
$mileageMbptotal
"; } return $content; diff --git a/src/MGRAST/lib/WebPage/MetagenomeOverview.pm b/src/MGRAST/lib/WebPage/MetagenomeOverview.pm index e2d284ba..c4d40bfd 100644 --- a/src/MGRAST/lib/WebPage/MetagenomeOverview.pm +++ b/src/MGRAST/lib/WebPage/MetagenomeOverview.pm @@ -146,7 +146,7 @@ sub output { my $mddb = $self->data('mddb'); my $user = $self->application->session->user; my $job_id = $job->job_id; - my $mg_link = "http://metagenomics.anl.gov/linkin.cgi?metagenome=$mgid"; + my $mg_link = $Conf::cgi_url."linkin.cgi?metagenome=$mgid"; # get project information my $project_link = ""; diff --git a/src/MGRAST/lib/WebPage/MetagenomeProject.pm b/src/MGRAST/lib/WebPage/MetagenomeProject.pm index fdb88cae..9f4c55b5 100644 --- a/src/MGRAST/lib/WebPage/MetagenomeProject.pm +++ b/src/MGRAST/lib/WebPage/MetagenomeProject.pm @@ -129,7 +129,7 @@ sub output { $self->{is_editor} = 1; } - my $proj_link = "http://metagenomics.anl.gov/linkin.cgi?project=".$self->{project_id}; + my $proj_link = $Conf::cgi_url."linkin.cgi?project=".$self->{project_id}; $html .= "

".$project->name.(($user and $user->has_right(undef, 'edit', 'user', '*')) ? " (ID ".$project->id.")": "")."

"; $html .= "

"; $html .= ""; diff --git a/src/MGRAST/lib/WebPage/MetagenomeSearch.pm b/src/MGRAST/lib/WebPage/MetagenomeSearch.pm index 91cff8e3..68da4ee6 100644 --- a/src/MGRAST/lib/WebPage/MetagenomeSearch.pm +++ b/src/MGRAST/lib/WebPage/MetagenomeSearch.pm @@ -235,6 +235,7 @@ sub output { my $scripts = qq~ +
Visibility".($project->public ? 'Public' : 'Private')."